{"type":"doc","content":[{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"toc","parameters":{"macroParams":{"_parentId":{"value":"26033979893"}},"macroMetadata":{"macroId":{"value":"678f981e-417c-48e1-a788-b7d9e9f178da"},"schemaVersion":{"value":"1"},"title":"Table of Contents"}},"localId":"1c158447-8bee-441a-9cda-d1319176184e"}},{"type":"heading","attrs":{"level":1},"content":[{"text":"Getting Started","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"How do I get an account?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"If you don’t already have an account, please fill out the ","type":"text","marks":[{"type":"strong"}]},{"text":"cluster application form","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"http://hpc.uconn.edu/storrs/account-application"}}]},{"text":". ","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Students and postdoctoral research associates who are requesting an account will need their advisor’s (PI’s) NetID so that we can verify their membership in the advisor’s research group. If you don’t know your advisor’s NetID, you can look it up on the ","type":"text"},{"text":"UConn PhoneBook","type":"text","marks":[{"type":"link","attrs":{"href":"https://phonebook.uconn.edu/lresults.php"}}]},{"text":".","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"How much does it cost to have an account with Storrs HPC?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"In general, it costs ","type":"text"},{"text":"nothing","type":"text","marks":[{"type":"strong"}]},{"text":". Basic access to the Storrs HPC is provided for","type":"text"},{"text":" free ","type":"text","marks":[{"type":"strong"}]},{"text":"to UConn students, staff, and faculty for research purposes. If you would like to have your jobs run more quickly, we offer priority access for a fee. See: ","type":"text"},{"text":"How do I get priority access to HPC resources?","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26033979893/FAQ#How-do-I-get-priority-access-to-HPC-resources?"}}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"What kind of data can be stored on the Storrs HPC?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":" ","type":"text"},{"text":"Short answer: ","type":"text","marks":[{"type":"strong"}]},{"text":"In general, data ","type":"text"},{"text":"cannot","type":"text","marks":[{"type":"strong"}]},{"text":" be stored on the Storrs HPC ","type":"text"},{"text":"if it contains","type":"text","marks":[{"type":"strong"}]},{"text":" any of the following:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"personal identifiable information (e.g., SSN, Passport Number, Drivers License Number, D.OB.)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"credit card and/or banking information (e.g., account numbers)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"university records for individual students (e.g., grades)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"personal health information / HIPAA-protected data (with few exceptions)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"data for which the data owner has specified data protection requirements that are not compatible with the Storrs HPC environment","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"data protected by UConn’s ","type":"text"},{"text":"Export Control Policy","type":"text","marks":[{"type":"link","attrs":{"href":"http://policy.uconn.edu/2015/12/16/export-control-policy/"}}]},{"text":" (e.g., DoD-related, requires a security clearance)","type":"text"}]}]}]},{"type":"rule"},{"type":"paragraph","content":[{"text":"Long Answer: ","type":"text","marks":[{"type":"strong"}]},{"text":"The Storrs HPC cluster cannot be used to generate or store data that is considered ","type":"text"},{"text":"Sensitive University Data","type":"text","marks":[{"type":"link","attrs":{"href":"http://security.uconn.edu/extended-list-of-confidential-data/"}}]},{"text":" or covered by the university's ","type":"text"},{"text":"Export Control Policy","type":"text","marks":[{"type":"link","attrs":{"href":"http://policy.uconn.edu/2015/12/16/export-control-policy/"}}]},{"text":"; for more info, see UConn’s ","type":"text"},{"text":"Data Classification Policy","type":"text","marks":[{"type":"link","attrs":{"href":"https://policy.uconn.edu/2012/06/21/data-classification-information-technology/"}}]},{"text":". The documents linked in the previous sentence provide more detail about each of those classifications. ","type":"text"}]},{"type":"paragraph","content":[{"text":"Data which may have been been classified as confidential or sensitive may be eligible for storage on Storrs HPC ","type":"text"},{"text":"if","type":"text","marks":[{"type":"underline"}]},{"text":" the data has been de-identified sufficiently that the data classification no longer applies. Any de/re-identification key (if one was used) cannot be transmitted or stored with the data within the HPC environment.","type":"text"}]},{"type":"paragraph","content":[{"text":"All data that is stored on the cluster is subject to these restrictions, and data that is not in compliance may be removed. If you have read the above documents and you’re still not sure how your data would be classified, please send an email to ","type":"text"},{"text":"security@uconn.edu","type":"text","marks":[{"type":"link","attrs":{"href":"mailto:security@uconn.edu"}}]},{"text":". ","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"How do I get priority access to HPC resources? ","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"Short answer","type":"text","marks":[{"type":"strong"}]},{"text":": Faculty can purchase priority access for 5 years if they pay the upfront cost for the nodes.","type":"text"}]},{"type":"paragraph","content":[{"text":"Long answer","type":"text","marks":[{"type":"strong"}]},{"text":": High-priority access is available under a “condo model,” where faculty are able to purchase semi-dedicated nodes which get made available to all users when there are unused compute cycles. Under the condo model, faculty researchers fund the capital equipment costs of individual compute nodes, while the university funds the operating costs of running these nodes for five years. Faculty who purchase compute nodes receive access to equivalent resources at a higher priority than other researchers. The faculty can designate others to receive access at the same priority level, such as their graduate students, postdoctoral researchers, etc. With priority access, computational jobs are moved higher in the queuing system, and in most cases begin execution within twelve hours, depending upon other FairShare factors. A priority user can utilize their resources indefinitely. All access to resources is managed through the cluster’s job scheduler. When a priority user is not using their assigned resources, the nodes are made available to all UConn researchers for general use.","type":"text"}]},{"type":"paragraph","content":[{"text":"Please note that priority users will not be given separate partitions. Instead, they will be given a custom QoS because the QoS governs access to priority resources (a.k.a. Trackable RESources, or TRes). ","type":"text"}]},{"type":"paragraph","content":[{"text":"If you are interested in investing in HPC resources, please fill out the ","type":"text"},{"text":"HPC Condo Request form","type":"text","marks":[{"type":"link","attrs":{"href":"https://hpc.uconn.edu/storrs/condo-application/"}}]},{"text":".","type":"text"}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Using the HPC","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"How do I log in to the Storrs HPC when I am off campus? ","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"Short answer","type":"text","marks":[{"type":"strong"}]},{"text":": First, you need to connect to UConn’s VPN. Then, you should be able to access the HPC. ","type":"text"}]},{"type":"paragraph","content":[{"text":"Long Answer","type":"text","marks":[{"type":"strong"}]},{"text":": The HPC Cluster only allows the connection of SSH from the campus-wide computers, for example: ","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"computers in the UConn libraries","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"computers in campus offices/labs","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"computers connected to UCONN-SECURE WiFi, etc.).","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"To connect to the HPC when you are off campus, you will first need to connect to the UConn Virtual Private Network (","type":"text"},{"text":"VPN","type":"text","marks":[{"type":"link","attrs":{"href":"https://vpn.uconn.edu%3B/"}}]},{"text":"). After connecting to the VPN, you will be able to log in to the HPC as you normally do.","type":"text"}]},{"type":"paragraph","content":[{"text":"For instructions on how to install, set up, and connect your personal device(s) to UConn’s VPN, please go to this ","type":"text"},{"text":"webpage","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/IKB/10907091023/Cisco%20AnyConnect%20VPN"}}]},{"text":". ","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"How do I check what node I am on? ","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"The node you are on will normally be shown next to your netID when you log in to the Storrs HPC. For instance, if Jonathan the Husky’s netID was jth10001, his terminal might look like this.","type":"text"}]},{"type":"codeBlock","content":[{"text":"[jth10001@login6 ~]$","type":"text"}]},{"type":"paragraph","content":[{"text":"This would tell us that Jonathan is on the node called “login6.” Another way to check what node you are on is to use the ","type":"text"},{"text":"hostname ","type":"text","marks":[{"type":"code"}]},{"text":"command. See below. ","type":"text"}]},{"type":"codeBlock","content":[{"text":"[jth10001@login6 ~]$ hostname\nlogin6","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"What programs am I allowed to run on the login nodes?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"Programs that are running on a login node (","type":"text"},{"text":"login.storrs.hpc.uconn.edu","type":"text","marks":[{"type":"code"}]},{"text":") without using the job scheduler are subject to certain restrictions. Programs that run on the login nodes for longer than 1 hour, use greater than 5% of CPU power, and/or use greater than 5% of RAM listed below may be throttled or terminated without notice.","type":"text"}]},{"type":"paragraph","content":[{"text":"Below is a list of programs that are allowed to run on the login node without restrictions:","type":"text"}]},{"type":"table","attrs":{"layout":"default","localId":"d288dfa1-3c0d-476a-b44b-cb6c71f964b2"},"content":[{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"awk","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"basemount","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"bash","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"bzip","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"chgrp","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"chmod","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"cmake","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"comsollauncher","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"cp","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"du","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"emacs","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"find","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"fort","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"gcc","type":"text"}]}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"gfortran","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"grep","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"gunzip","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"gzip","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"icc","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"ifort","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"jservergo","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"less","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"ls","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"make","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"more","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"mv","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"nano","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"ncftp","type":"text"}]}]}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"nvcc","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"perl","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"rm","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"rsync","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"ruby","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"setfacl","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"sftp","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"smbclient","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"ssh","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"tail","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"tar","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"ukbfetch","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"vim","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"wget","type":"text"}]}]}]}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"How do I access old data after I leave the university?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"Short answer: ","type":"text","marks":[{"type":"strong"}]},{"text":" Email us at ","type":"text"},{"text":"hpc@uconn.edu","type":"text","marks":[{"type":"link","attrs":{"href":"mailto:hpc@uconn.edu"}}]},{"text":". We will set up an affiliate account so you can access those files.","type":"text"}]},{"type":"paragraph","content":[{"text":"Long answer: ","type":"text","marks":[{"type":"strong"}]},{"text":"When folks leave the university, their netIDs are deactivated and access to their Storrs HPC resources and files are (typically) transferred to their supervisors. To regain access, Storrs HPC staff need to create an account affiliated with their former supervisor. ","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"How do I install specific R packages?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"Since installing R packages is important to so many HPC users' research, we have created a brief guide on the installation of R packages which is linked ","type":"text"},{"text":"here","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26033587623/R+Guide#Install-an-R-package"}}]},{"text":".","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"How do I install specific python libraries?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"We have created a brief guide on the installation of python libraries which can be found ","type":"text"},{"text":"here","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26033914173/Python+Guide#Installing-Python-libraries"}}]},{"text":".","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"How can I check for available resources before submitting a job?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"Long wait times for your jobs? Errors about unavailable resources? We’ve been there and understand how frustrating it can be for jobs to take a long time to run. It’s an unfortunate consequence of having such a strong computational research community at UConn. LOTS of incredible research happens here, but it also means that there are LOTS of people competing for resources. There’s no getting around that problem, but there are a couple of steps we can take to increase the odds that our jobs get on ASAP.","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Check what resources are available before we submit a job. (","type":"text"},{"text":"And then","type":"text","marks":[{"type":"em"}]},{"text":")","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Target our submission to those available resources. ","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"This FAQ will offer guidance on how to do both of those things.","type":"text"}]},{"type":"rule"},{"type":"paragraph","content":[{"text":"Checking for Available Resources","type":"text","marks":[{"type":"underline"},{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"The below ","type":"text"},{"text":"nodeinfo","type":"text","marks":[{"type":"code"}]},{"text":" command will give you a high-level view of what nodes are fully available (“idle”), in partial use (“mix”), not available (“alloc”. for allocated), or otherwise in need of maintenance (all other states besides idle, mix, or alloc). The nodes will be broken down in order by partition. ","type":"text"}]},{"type":"codeBlock","content":[{"text":"nodeinfo","type":"text"}]},{"type":"paragraph","content":[{"text":"The output for that command will look like this, but it will be much longer and provide info on every partition (not just class and debug).","type":"text"}]},{"type":"codeBlock","content":[{"text":"PARTITION NODES STATE TIMELIMIT CPUS GRES MEMORY ACTIVE_FEATURES NODELIST\nclass 1 inval 4:00:00 36 gpu:3 191954 gpu,v100,skylake gpu05\nclass 1 maint* 4:00:00 128 (null) 515404 epyc128,cpuonly cn563\nclass 3 drain$ 4:00:00 36 (null) 191845+ skylake,cpuonly cn[360-361,363]\nclass 8 maint 4:00:00 36 (null) 191845+ skylake,cpuonly cn[362,365-366,368-372]\nclass 1 maint 4:00:00 128 (null) 515404 epyc128,cpuonly cn561\nclass 1 comp* 4:00:00 128 (null) 515404 epyc128,cpuonly cn564\nclass 3 drng 4:00:00 128 (null) 515404 epyc128,cpuonly cn[560,565,569]\nclass 6 alloc 4:00:00 128 (null) 515404 epyc128,cpuonly cn[559,562,566-568,570]\ndebug 1 maint* 30:00 128 (null) 515404 epyc128,cpuonly cn563\n\n... (etc.)","type":"text"}]},{"type":"paragraph","content":[{"text":"The above command gives us an overarching picture of usage on the cluster, and from there, we can use a more targeted command to get more information on individual nodes within a partition, like how many cores or GPUs are in use and how many are available. The base ","type":"text"},{"text":"sinfo","type":"text","marks":[{"type":"code"}]},{"text":" command is below and it targets the ","type":"text"},{"text":"priority-gpu","type":"text","marks":[{"type":"code"}]},{"text":" partition but we could amend it to target any other partition, like ","type":"text"},{"text":"hi-core","type":"text","marks":[{"type":"code"}]},{"text":" for example. ","type":"text"}]},{"type":"codeBlock","content":[{"text":"sinfo -p priority-gpu -t idle,mix -o%10n%20C%10G%10t%20R%b\nHOSTNAMES CPUS(A/I/O/T) GRES STATE PARTITION ACTIVE_FEATURES\ngpu06 13/23/0/36 gpu:1 mix priority-gpu location=local,gpu,v100,skylake\ngpu20 36/28/0/64 gpu:3 mix priority-gpu location=local,epyc64,a100,gpu\ngpu21 25/39/0/64 gpu:3 mix priority-gpu location=local,epyc64,a100,gpu\ngpu22 3/61/0/64 gpu:3 mix priority-gpu location=local,epyc64,a100,gpu\ngpu23 33/31/0/64 gpu:1 mix priority-gpu location=local,epyc64,a100,gpu\ngtx02 2/18/0/20 gpu:3 mix priority-gpu location=local,gpu,gtx,skylake\ngtx03 12/8/0/20 gpu:3 mix priority-gpu location=local,gpu,gtx,skylake\ngtx08 1/19/0/20 gpu:2 mix priority-gpu location=local,gpu,gtx,skylake\ngtx11 2/18/0/20 gpu:2 mix priority-gpu location=local,gpu,gtx,skylake\ngtx15 28/4/0/32 gpu:8 mix priority-gpu location=local,gpu,rtx,skylake\ngtx09 0/20/0/20 gpu:2 idle priority-gpu location=local,gpu,gtx,skylake\n","type":"text"}]},{"type":"paragraph","content":[{"text":"The column titled “CPUS (A/I/O/T)” tells us how many cores are available. “A” stands for Allocated, “I” stands for Idle, and “T” stands for Total. (“O” stands for Other but you can ignore that column) Since there are 39 cores in the “Idle” column for GPU21, that means 39 cores are available to use. But all 3 of the GPUs on GPU21 are in use so we can’t use any GPUs on that node. So, that gives us an idea of the resources. If I only needed cores and no GPUs, I could target GPU21. ","type":"text"}]},{"type":"paragraph","content":[{"text":"You can also use the following to see available recources for a particular partition in a simpler format. For example:","type":"text"}]},{"type":"codeBlock","content":[{"text":"sinfo -p priority -o%C | sed 's/CPUS//; s/A/Alloc/; s/I/Idle/; s/O/Other/; s/T/Total/'| tr \"/\" \"\\t\" | tr -d \"()\"\nAlloc Idle Other Total\n4667 8664 705 14036","type":"text"}]},{"type":"paragraph","content":[{"text":"In summary, these two commands can give us a picture of what partitions have resources available, and then what resources are available on individual nodes within that partition. ","type":"text"}]},{"type":"rule"},{"type":"paragraph","content":[{"text":"Targeting a specific partition","type":"text","marks":[{"type":"underline"},{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"The next step is submitting a job targeting a specific partition. If you’re not sure how to target a specific partition, please visit our ","type":"text"},{"text":"SLURM Guide","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26032963685/SLURM+Guide#Job-Submission"}}]},{"text":" where you will see examples of submission scripts that target different partitions and architectures. Another key part of targeting a specific partition is knowing what partitions you are allowed to use and what “","type":"text"},{"text":"account","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26449379370/SLURM+Cheatsheet#Job-Submission"}}]},{"text":"” and “","type":"text"},{"text":"QOS","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26449379370/SLURM+Cheatsheet#Job-Submission"}}]},{"text":"” you must use to access them. To check what partitions you’re allowed to use and how to access them you can use this command.","type":"text"}]},{"type":"codeBlock","content":[{"text":"acctinfo`whoami`\n User Account Partition QOS \n---------- ---------- -------------------- -------------------- \nnet10004 jth10001 hi-core general \nnet10004 jth10001 general-gpu general \nnet10004 jth10001 debug general \nnet10004 jth10001 lo-core general \nnet10004 jth10001 general general\nnet10004 jth10001 priority-gpu jth10001gpu","type":"text"}]},{"type":"paragraph","content":[{"text":"This tells me that I have access to 6 partitions. To access the priority-gpu partition, I need to include the below three flags in the #SBATCH header of my submission script. This will be different for every individual so you will have to modify this with the partitions you have access to and the account and QOS that are associated with your account. ","type":"text"}]},{"type":"codeBlock","content":[{"text":"#SBATCH -p priority-gpu # partition I want to access\n#SBATCH -A jth10001 # account I am associated with\n#SBATCH -q jth10001gpu # QOS needed to access partition","type":"text"}]},{"type":"paragraph","content":[{"text":"If you have further questions about how to check what resources are available and how to target them, please feel free to contact the Storrs HPC admins by sending an email to ","type":"text"},{"text":"hpc@uconn.edu","type":"text","marks":[{"type":"link","attrs":{"href":"mailto:hpc@uconn.edu"}}]},{"text":".","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"How does the HPC decide which jobs run first?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"HPCs have a lot of similarities with airports. Let’s talk about how:","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Security and Access","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Accessing an HPC is a bit like entering an airport. To get inside an airport, you have to show your ID and go through security. Similarly, you need to ","type":"text"},{"text":"both ","type":"text","marks":[{"type":"strong"}]},{"text":"have an account","type":"text","marks":[{"type":"underline"}]},{"text":" and ","type":"text"},{"text":"use your password","type":"text","marks":[{"type":"underline"}]},{"text":" (or an ssh key) to access the HPC. ","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center","width":730,"widthType":"pixel"},"content":[{"type":"media","attrs":{"__fileMimeType":"application/octet-stream","width":3960,"id":"UNKNOWN_MEDIA_ID","collection":"","type":"file","height":3240}},{"type":"caption","content":[{"text":" image by ","type":"text"},{"text":"macrovector ","type":"text","marks":[{"type":"link","attrs":{"href":"https://www.freepik.com/free-vector/airport-indoor-isometric-illustration_6375483.htm#query=airport%20security&position=13&from_view=keyword&track=ais&uuid=bbd144e2-29ed-4a41-920c-59366613eb7e#page=1&query=a&from_query=undefined&position=0&from_view=keyword&track=ais&uuid=bbd144e2-29ed-4a41-920c-59366613eb7e"}}]},{"text":"on Freepik","type":"text"}]}]},{"type":"paragraph","content":[{"text":"Once you get inside, you hang around the terminal until your flight starts boarding. The HPC equivalent of the terminal is called the ","type":"text"},{"text":"login node","type":"text","marks":[{"type":"strong"}]},{"text":". There are some basic things you can do on login nodes---move files, edit documents, etc.---but you shouldn’t run any really computationally programs or analyses on the login nodes. It’d be like blatantly cutting in front of everyone in line at the airport Starbuck’s. Not only would it be disrespectful to everyone else in line (i.e., ","type":"text"},{"text":"on the login node","type":"text","marks":[{"type":"em"}]},{"text":"), when the Starbuck’s staff (i.e., ","type":"text"},{"text":"Storrs HPC Admins","type":"text","marks":[{"type":"em"}]},{"text":") saw what happened, that person would be kicked out of the cafe.","type":"text"}]},{"type":"paragraph","content":[{"text":"Intense computational programs or analyses, a.k.a. ","type":"text"},{"text":"jobs","type":"text","marks":[{"type":"strong"}]},{"text":", should be run on ","type":"text"},{"text":"compute nodes","type":"text","marks":[{"type":"strong"}]},{"text":". In this analogy, jobs are ","type":"text"},{"text":"groups of passengers","type":"text","marks":[{"type":"em"}]},{"text":" (e.g., families) and compute nodes are planes. Just as planes have a limited number of seats, compute nodes have a limited number of cores (a.k.a. CPUs). Jobs can only get onto compute nodes that have enough cores (i.e., “seats”) available. ","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center","width":730,"widthType":"pixel"},"content":[{"type":"media","attrs":{"__fileMimeType":"application/octet-stream","width":8000,"id":"UNKNOWN_MEDIA_ID","collection":"","type":"file","height":4000}},{"type":"caption","content":[{"text":"Image by ","type":"text"},{"text":"pch.vector","type":"text","marks":[{"type":"link","attrs":{"href":"https://www.freepik.com/free-vector/passengers-with-luggage-airport-flat-illustration_11235789.htm#query=airport%20security&position=28&from_view=keyword&track=ais&uuid=bbd144e2-29ed-4a41-920c-59366613eb7e"}}]},{"text":" on Freepik","type":"text"}]}]},{"type":"paragraph","content":[{"text":"People who buy tickets (","type":"text"},{"text":"submit ","type":"text","marks":[{"type":"strong"},{"type":"em"}]},{"text":"jobs","type":"text","marks":[{"type":"em"}]},{"text":") ahead of time generally have their seats reserved ahead of people who just show up at the gate looking for a seat (","type":"text"},{"text":"requesting an","type":"text","marks":[{"type":"em"}]},{"text":" interactive job","type":"text","marks":[{"type":"strong"},{"type":"em"}]},{"text":"). But there are exceptions. When a flight is overbooked (","type":"text"},{"text":"more cores requested than the HPC has available","type":"text","marks":[{"type":"em"}]},{"text":"), people who are part of an airline’s frequent flyer program (","type":"text"},{"text":"have ","type":"text","marks":[{"type":"em"}]},{"text":"priority access","type":"text","marks":[{"type":"strong"},{"type":"em"}]},{"text":" on the HPC","type":"text","marks":[{"type":"em"}]},{"text":") get first dibs on (","type":"text"},{"text":"HPC resources","type":"text","marks":[{"type":"em"}]},{"text":"). ","type":"text"}]},{"type":"paragraph","content":[{"text":"Okay, this analogy works well for how individuals relate to the HPC, but we have to use a different analogy to understand the structure of HPCs and how they operate as a whole.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Structure and Organization","type":"text"}]},{"type":"paragraph","content":[{"text":"To understand the broader structure of HPCs and how they work, we can look at how airports are organized. All major airports have an air traffic control tower and an air traffic controller (ATC) working inside it. The HPC’s equivalent of the ATC tower is called the ","type":"text"},{"text":"head node","type":"text","marks":[{"type":"strong"}]},{"text":" and the role of the ATC is played by a program called ","type":"text"},{"text":"SLURM","type":"text","marks":[{"type":"strong"}]},{"text":". The ATC’s (","type":"text"},{"text":"SLURM’s","type":"text","marks":[{"type":"em"}]},{"text":") main job is directing which planes (","type":"text"},{"text":"jobs","type":"text","marks":[{"type":"strong"},{"type":"em"}]},{"text":") get to use the runways (","type":"text"},{"text":"compute nodes","type":"text","marks":[{"type":"em"}]},{"text":") because there are usually more planes flying in the air (","type":"text"},{"text":"running ","type":"text","marks":[{"type":"strong"},{"type":"em"}]},{"text":"jobs)","type":"text","marks":[{"type":"em"}]},{"text":" and waiting on the ground (","type":"text"},{"text":"pending ","type":"text","marks":[{"type":"strong"},{"type":"em"}]},{"text":"jobs","type":"text","marks":[{"type":"em"}]},{"text":") than there are runways (","type":"text"},{"text":"nodes","type":"text","marks":[{"type":"em"}]},{"text":") available. ","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center","width":730,"widthType":"pixel"},"content":[{"type":"media","attrs":{"__fileMimeType":"image/jpeg","width":6500,"alt":"35049-20240404-203331.jpg","id":"UNKNOWN_MEDIA_ID","collection":"","type":"file","height":4179}},{"type":"caption","content":[{"text":"Image by ","type":"text"},{"text":"macrovector","type":"text","marks":[{"type":"link","attrs":{"href":"https://www.freepik.com/free-vector/airport-illustration_9434057.htm#&position=0&from_view=search&track=ais&uuid=c535cea0-0956-4761-8962-02c113e84f99"}}]},{"text":" on Freepik","type":"text"}]}]},{"type":"paragraph","content":[{"text":"The ATC (","type":"text"},{"text":"SLURM","type":"text","marks":[{"type":"em"}]},{"text":") takes many things into account when deciding which planes (","type":"text"},{"text":"jobs","type":"text","marks":[{"type":"em"}]},{"text":") get to use a given runway (","type":"text"},{"text":"node","type":"text","marks":[{"type":"em"}]},{"text":") next. Here a few that are similar to HPCs.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"how long a plane (","type":"text"},{"text":"job","type":"text","marks":[{"type":"em"}]},{"text":") has been waiting on the ground (","type":"text"},{"text":"pending in the job queue","type":"text","marks":[{"type":"em"}]},{"text":")","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"the size of the plane (","type":"text"},{"text":"how many cores the job is requesting","type":"text","marks":[{"type":"em"}]},{"text":") ","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"In general, SLURM will let a job that has been waiting for 6 hours will get on the next open node before a job that has been waiting for 1 hour. But if a job is massive, smaller jobs may get on before it.","type":"text"}]},{"type":"paragraph","content":[{"text":"Here’s where SLURM differs from an airport’s ATC. Access to airport runways generally operate on a first come-first served basis, but SLURM adds another consideration to prevent a single HPC user from monopolizing all of the HPC’s resources. It takes into account the number (and size) of jobs a given user has submitted recently. A user who has submitted thousands of jobs in the last week will be pushed down the list to give all users fair, equitable access to HPC resources. ","type":"text"}]},{"type":"paragraph","content":[{"text":"The last important consideration for an HPC which doesn’t generally apply to airports is that users can buy priority access to HPC resources. Priority access is like TSA PreCheck. Priority users still wait in line, but it’s a much shorter line and they tend to get through faster.","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"When will my pending job start running?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"You can use the following command to see SLURM’s estimate of the start time:","type":"text"}]},{"type":"codeBlock","content":[{"text":"squeue -j --start","type":"text"}]},{"type":"paragraph","content":[{"text":"This is only an estimate, and in most cases will not be the actual start times of the jobs.","type":"text"}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Troubleshooting problems","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Why did my job fail?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"There are many reasons a job may fail. A good first step is to use the ","type":"text"},{"text":"shist","type":"text","marks":[{"type":"code"}]},{"text":" command to check the ","type":"text"},{"text":"ExitCode","type":"text","marks":[{"type":"code"}]},{"text":" SLURM gave it. The command follows this format: ","type":"text"},{"text":"shist --starttime YYYY-MM-DD","type":"text","marks":[{"type":"code"}]},{"text":".","type":"text"}]},{"type":"paragraph","content":[{"text":"Here’s an example of the output for a job that failed immediately with an ","type":"text"},{"text":"ExitCode","type":"text","marks":[{"type":"code"}]},{"text":" of 1. ","type":"text"}]},{"type":"codeBlock","content":[{"text":"\nJobID Partition QOS JobName User State Elapsed NNodes NCPUS NodeList ExitCode End \n------------ ---------- ---------- ---------- --------- ---------- ---------- -------- ---------- --------------- -------- -------------------\n73088 priority-+ erm12009g+ submit_rx jdt10005 FAILED 00:00:00 1 32 gtx21 1:0 2022-11-17T10:05:34 ","type":"text"}]},{"type":"paragraph","content":[{"text":"The ","type":"text"},{"text":"ExitCode","type":"text","marks":[{"type":"code"}]},{"text":" of a job will be a number between 0 and 255. An ","type":"text"},{"text":"ExitCode","type":"text","marks":[{"type":"code"}]},{"text":" of 0 means that—as far as SLURM is concerned—the job ran and was completed properly. Any non-zero ","type":"text"},{"text":"ExitCode","type":"text","marks":[{"type":"code"}]},{"text":" will indicate that the job failed. One could then search the ","type":"text"},{"text":"ExitCode","type":"text","marks":[{"type":"code"}]},{"text":" on Google to investigate what SLURM thinks caused the job to fail. Sometimes this is helpful but not always. Either way, take note of what you find for future reference. Common ExitCodes are listed below for reference:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"0 → success","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"non-zero → failure","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Exit code 1 indicates a general failure","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Exit code 2 indicates incorrect use of shell builtins","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Exit codes 3-124 indicate some error in job (check software exit codes)","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Exit code 125 indicates out of memory","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Exit code 126 indicates command cannot execute","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Exit code 127 indicates command not found","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Exit code 128 indicates invalid argument to exit","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Exit codes 129-192 indicate jobs terminated by Linux signals","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"For these, subtract 128 from the number and match to signal code","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Enter ","type":"text"},{"text":"kill -l","type":"text","marks":[{"type":"code"}]},{"text":" to list signal codes","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Enter ","type":"text"},{"text":"man signal","type":"text","marks":[{"type":"code"}]},{"text":" for more information","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Please note:","type":"text","marks":[{"type":"em"}]},{"text":" when a signal was responsible for a job or step's termination, the signal number will be displayed after the exit code, delineated by a colon(:).","type":"text"}]}]}]}]}]},{"type":"paragraph","content":[{"text":"The next clue to investigate is the ","type":"text"},{"text":"NodeList","type":"text","marks":[{"type":"code"}]},{"text":" column. Sometimes a job fails because there is something wrong with the ","type":"text"},{"text":"compute node","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26033979511/Glossary#HPC-Terminology"}}]},{"text":" our job was run on. If the compute node is the problem (and Storrs HPC staff haven’t fixed it already), the job should fail again with the same ","type":"text"},{"text":"ExitCode","type":"text","marks":[{"type":"code"}]},{"text":". We can submit our job specifically to that same node to see if the job fails again. Try adding this to the #SBATCH header of your script to target a specific node. Here, we target ","type":"text"},{"text":"gtx21","type":"text","marks":[{"type":"code"}]},{"text":" because that was the node listed in the ","type":"text"},{"text":"NodeList","type":"text","marks":[{"type":"code"}]},{"text":" column above. ","type":"text"}]},{"type":"codeBlock","content":[{"text":"#SBATCH --nodelist=gtx21","type":"text"}]},{"type":"paragraph","content":[{"text":"Once you see the job has failed multiple times on the same node but does not fail on other nodes, then you can feel confident that a faulty node is a likely cause. Please ","type":"text"},{"text":"submit a help request","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26474020886/Steps+to+Successfully+Troubleshoot+on+the+HPC"}}]},{"text":" to Storrs HPC including a screenshot from the ","type":"text"},{"text":"shist","type":"text","marks":[{"type":"code"}]},{"text":" output.","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"My jobs are failing due to insufficient memory. Or with an “out of memory” or “OOM” error. Why is this happening? And how do I fix this? ","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"Short answer","type":"text","marks":[{"type":"strong"}]},{"text":": If you received this error, your job most likely failed because the amount of memory (RAM) it needed was larger than the default. We can request more memory for your job using the ","type":"text"},{"text":"--mem","type":"text","marks":[{"type":"code"}]},{"text":" or ","type":"text"},{"text":"--mem-per-cpu ","type":"text","marks":[{"type":"code"}]},{"text":"flag","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26449379370/SLURM+Cheatsheet"}}]},{"text":". ","type":"text"}]},{"type":"paragraph","content":[{"text":"Long answer:","type":"text","marks":[{"type":"strong"}]},{"text":" There are several reasons a job may fail from insufficient memory. As of January 2023, the default amount of memory available per CPU is 2 gigabytes. The default ","type":"text"},{"text":"mem-per-cpu","type":"text","marks":[{"type":"code"}]},{"text":" of 2 GB is okay for many users, but not for all. Users who receive OOM errors need to request more memory when submitting or initiating jobs. Those users can easily override the default using the ","type":"text"},{"text":"-mem-per-cpu","type":"text","marks":[{"type":"code"}]},{"text":" flag in your submission script. The new line would look something like this:","type":"text"}]},{"type":"codeBlock","content":[{"text":"#SBATCH -n 2 # asks for 2 cores\n#SBATCH --mem-per-cpu=3G # asks for 3 GB or RAM per core, or 6 GB total","type":"text"}]},{"type":"paragraph","content":[{"text":"Adding this line will tell SLURM to use 3 gigabytes of RAM per CPU you request. That means if we ask for 2 cores (-n 2), then we’ll be allocated 6 gigabytes of RAM total. Please note that the ","type":"text"},{"text":"--mem-per-cpu","type":"text","marks":[{"type":"code"}]},{"text":" flag must be used with the ","type":"text"},{"text":"-n","type":"text","marks":[{"type":"code"}]},{"text":" flag specifying the number of cores you want. Alternatively, users can use the ","type":"text"},{"text":"--mem","type":"text","marks":[{"type":"code"}]},{"text":" flag to specify the total amount of RAM they need regardless of how many CPUs are requested. ","type":"text"}]},{"type":"codeBlock","content":[{"text":"#SBATCH --mem=5G # asks for 5 gigabytes total","type":"text"}]},{"type":"paragraph","content":[{"text":"We encourage users to please adjust the ","type":"text"},{"text":"--mem-per-cpu","type":"text","marks":[{"type":"code"}]},{"text":" or ","type":"text"},{"text":"--mem","type":"text","marks":[{"type":"code"}]},{"text":" flags in a step-wise fashion. First, we try 3 gigabytes, then 4, then 5, etc. until our jobs start running without failing from memory errors. That strategy helps ensure that every HPC user's jobs get on quickly and run efficiently. For more info on fisbatch, srun, and #SBATCH flags, see this ","type":"text"},{"text":"link","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26449379370/SLURM+Cheatsheet"}}]},{"text":".","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"My jobs are failing due to Timeout. I do not have access to ","type":"text"},{"text":"priority","type":"text","marks":[{"type":"em"}]},{"text":"; how can I resume a job after it times out? ","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"Short answer","type":"text","marks":[{"type":"strong"}]},{"text":": Once the job is canceled by SLURM due to timeout, it cannot be resumed from that point because SLURM sets the exit code to “0” which denotes job completion. As far as SLURM is concerned, the job is now complete, with no state to resume from.","type":"text"}]},{"type":"paragraph","content":[{"text":"Long answer:","type":"text","marks":[{"type":"strong"}]},{"text":" One thing you can try is to use the ","type":"text"},{"text":"timeout ","type":"text","marks":[{"type":"em"}]},{"text":"command to stop your program just before SLURM does. You can tell from the return code if the timeout was reached or not. It should set exit code “124”. If so, you can then requeue it with ","type":"text"},{"text":"scontrol","type":"text","marks":[{"type":"em"}]},{"text":". Try the following:","type":"text"}]},{"type":"paragraph","content":[{"text":"In your submission script, add the following:","type":"text"}]},{"type":"codeBlock","content":[{"text":"#SBATCH --open-mode=append\n#SBATCH --time=12:00:00","type":"text"}]},{"type":"paragraph","content":[{"text":"Then, use the ","type":"text"},{"text":"timeout ","type":"text","marks":[{"type":"em"}]},{"text":"command to call your program:","type":"text"}]},{"type":"codeBlock","content":[{"text":"timeout 11h ./your_program\nif [[ $? == 124 ]]; then\n scontrol requeue $SLURM_JOB_ID\nfi","type":"text"}]},{"type":"paragraph","content":[{"text":"Note","type":"text","marks":[{"type":"strong"},{"type":"em"}]},{"text":": The ","type":"text"},{"text":"--open-mode=append","type":"text","marks":[{"type":"code"}]},{"text":" ensures the output of each run is appended to the file specified by ","type":"text"},{"text":"#SBATCH --output=","type":"text","marks":[{"type":"code"}]},{"text":" to preserve the previous run’s output in the same file.","type":"text"}]},{"type":"paragraph","content":[{"text":"Disclaimer","type":"text","marks":[{"type":"em"}]},{"text":":","type":"text","marks":[{"type":"strong"},{"type":"em"}]},{"text":" This is untested on Storrs HPC; however, it should work as long as everything else is working correctly.","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"My job is not running. It says “JobHeldUser” and its state is “SE.” Why is this happening? And how do I get my job running again?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"Short answer","type":"text","marks":[{"type":"strong"}]},{"text":": Your job is being “held.” To release the job and re-submit it to the job queue you can use the ","type":"text"},{"text":"scontrol release {JOBID}","type":"text","marks":[{"type":"code"}]},{"text":" command.","type":"text"}]},{"type":"paragraph","content":[{"text":"Long Answer","type":"text","marks":[{"type":"strong"}]},{"text":": Your job failed. We have a separate FAQ on figuring out why a job failed ","type":"text"},{"text":"here","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26033979893/FAQ#Why-did-my-job-fail?"}}]},{"text":", but here we will focus on why your job is being held. When jobs fail, they used to be automatically re-queued. This was a problem for a number of users because re-running the job would overwrite their previous data. In January 2024, we re-configured SLURM to prevent this problem. Now, when jobs fail, they are not immediately re-queued. Instead, the jobs will be “held” from the queue until the submitting user “releases” those jobs back into the queue. This change prevents jobs from requeueing automatically and allows users to make a conscious choice to re-queue their jobs. You can re-queue jobs using the below commands:","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"To release a single job","type":"text"}]},{"type":"codeBlock","content":[{"text":"scontrol release {JOBID}","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"To release multiple jobs","type":"text"}]},{"type":"codeBlock","content":[{"text":"scontrol release {JOBID_1},{JOBID_2},{JOBID_3}","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"To release all jobs with a given job name","type":"text"}]},{"type":"codeBlock","content":[{"text":"scontrol release jobname={JOBNAME}","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"To release all of your held jobs","type":"text"}]},{"type":"codeBlock","content":[{"text":"squeue --me | grep ' SE ' | awk '{print $1}' | xargs -n1 scontrol release","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"If you release your jobs into the queue and they keep ending up back in the “held” state, that is an indication that there may be something failing within your submission script in which case you should cancel your jobs and start troubleshooting. Please note that jobs that are left in the queue with the “SE” state will be canceled after seven days. ","type":"text"}]},{"type":"paragraph","content":[{"text":"Please feel free to contact us at ","type":"text"},{"text":"hpc@uconn.edu","type":"text","marks":[{"type":"link","attrs":{"href":"mailto:hpc@uconn.edu"}}]},{"text":" with any questions or concerns.","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Some of my files were deleted. Is it possible to recover them? If so, how?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"Short answer","type":"text","marks":[{"type":"strong"}]},{"text":": Possibly, it depends on where the files were stored and how long ago they were deleted. Send us an email at ","type":"text"},{"text":"hpc@uconn.edu","type":"text","marks":[{"type":"link","attrs":{"href":"mailto:hpc@uconn.edu"}}]},{"text":" explaining the situation so we can discuss options. ","type":"text"}]},{"type":"paragraph","content":[{"text":"Long Answer","type":"text","marks":[{"type":"strong"}]},{"text":": Files stored in the /shared/ and /home/ directories are backed up daily at 5:00 am and the data is stored for 30 days. If the accidentally deleted files were stored within either of those directories, and you contact us in under 30 days, then there’s a chance we will be able to help you restore those files. Please contact us at ","type":"text"},{"text":"hpc@uconn.edu","type":"text","marks":[{"type":"link","attrs":{"href":"mailto:hpc@uconn.edu"}}]},{"text":" so we can discuss a way forward.","type":"text"}]},{"type":"paragraph","content":[{"text":"Unfortunately, if the files that were deleted had been stored on /scratch/, we will not be able to restore them. The /scratch/ directory is ","type":"text"},{"text":"not","type":"text","marks":[{"type":"underline"}]},{"text":" backed up ever. If you’re reading this after accidentally deleting files from scratch, we understand that this news can come as a huge blow. We wish that we could back up /scratch/, but we only have enough room to back /shared/ and /home/. We cannot help you fix the current situation, but there are steps we can take to prevent this from happening again. You can follow these links for more info. on HPC ","type":"text"},{"text":"data storage","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26034012236/Data+Storage+Guide"}}]},{"text":", ","type":"text"},{"text":"backing up files","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26033816276/Data+Backup"}}]},{"text":", or using ","type":"text"},{"text":"Globus","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26033653003/Globus+Connect"}}]},{"text":".","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Why am I suddenly getting a “Permission denied” error when I try to ssh into the HPC? ","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"Short answer","type":"text","marks":[{"type":"strong"}]},{"text":": One of the login nodes is most likely not working properly. Try to ssh into any of the three login nodes directly. ","type":"text"}]},{"type":"paragraph","content":[{"text":"Long Answer","type":"text","marks":[{"type":"strong"}]},{"text":": When you ssh into ","type":"text"},{"text":"login.storrs.hpc.uconn.edu","type":"text","marks":[{"type":"link","attrs":{"href":"http://login.storrs.hpc.uconn.edu"}}]},{"text":", you are directed to one of our three login nodes (login4, login5, or login6). Occasionally, one of these three login nodes will become faulty. If you ssh into the cluster and your account is directed to the faulty node, then you may be given a “Permission denied” error message. If you experience this problem, we recommend you try to ssh directly into one of the login nodes. Here are the three commands one can use to login into our three login nodes. Please replace netID with your own netID.","type":"text"}]},{"type":"paragraph","content":[{"text":"login4","type":"text","marks":[{"type":"strong"}]}]},{"type":"codeBlock","content":[{"text":"ssh -Y netID@login4.storrs.hpc.uconn.edu","type":"text"}]},{"type":"paragraph","content":[{"text":"login5","type":"text","marks":[{"type":"strong"}]}]},{"type":"codeBlock","content":[{"text":"ssh -Y netID@login5.storrs.hpc.uconn.edu","type":"text"}]},{"type":"paragraph","content":[{"text":"login6","type":"text","marks":[{"type":"strong"}]}]},{"type":"codeBlock","content":[{"text":"ssh -Y netID@login6.storrs.hpc.uconn.edu","type":"text"}]},{"type":"paragraph","content":[{"text":"If one of these allows you to log in but another gives you a permission denied error, then we can be sure that there is something wrong with one of the login nodes. If you have the time, we’ll ask that you please send us a screenshot of the login node that is giving you a problem so we can tend to any problems on that faulty node. This will help us ensure that this problem is fixed as soon as possible. ","type":"text"}]},{"type":"paragraph","content":[{"text":"If you receive this “Permission denied” error when ssh-ing directly into all three login nodes, then the problem may be with your netID. You may have to reset your netID password which can be done at this ","type":"text"},{"text":"link","type":"text","marks":[{"type":"link","attrs":{"href":"https://netid.uconn.edu/reset_password.php"}}]},{"text":".","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Why do I get a GLIBC Error when trying to load certain modules?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"Short answer","type":"text","marks":[{"type":"strong"}]},{"text":": Some modules can only be run on certain architectures. Try loading that module on a node with the Epyc or Skylake architectures. ","type":"text"}]},{"type":"paragraph","content":[{"text":"Long Answer","type":"text","marks":[{"type":"strong"}]},{"text":": These GLIBC errors often happen because the software you’re trying to load expects to find a newer GLIBC library than what’s available in your current node’s Red Hat Enterprise Linux (RHEL) version. Older architectures sometimes have older RHEL versions and therefore older GLIBC libraries. Newer architectures typically have newer RHEL versions and GLIBC libraries.","type":"text"}]},{"type":"paragraph","content":[{"text":"Switching to a newer or different architecture may resolve this GLIBC error. See the following ","type":"text"},{"text":"guide","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26032963610/SLURM+Partitions+and+Job+Scheduling"}}]},{"text":" for instructions on how to target specific architectures. ","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"How do I fix an error that says, “Can't open display, Failed initializing GUI, exiting?”","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"Short answer","type":"text","marks":[{"type":"strong"}]},{"text":": This is an X-forwarding error. The most common fix is to enable X-forwarding when you ssh into the HPC using the ","type":"text"},{"text":"-X","type":"text","marks":[{"type":"code"}]},{"text":" or ","type":"text"},{"text":"-Y","type":"text","marks":[{"type":"code"}]},{"text":" flag. But there are other problems which can cause this too.","type":"text"}]},{"type":"codeBlock","content":[{"text":"ssh -Y netID@hpc2.storrs.hpc.uconn.edu","type":"text"}]},{"type":"paragraph","content":[{"text":"Long Answer","type":"text","marks":[{"type":"strong"}]},{"text":": X-forwarding allows programs being run on the HPC to be opened up in a GUI on our local machines. It is convenient and nice to work with but it can take a bit of effort to set up if you are working on a Mac or Windows device. Linux users have it easy because all they normally have to do is use the ","type":"text"},{"text":"-X","type":"text","marks":[{"type":"code"}]},{"text":" or ","type":"text"},{"text":"-Y","type":"text","marks":[{"type":"code"}]},{"text":" flag we referred to above. For users, of Windows or Mac devices, we have written up a more in-depth guide which you can find ","type":"text"},{"text":"here","type":"text","marks":[{"type":"link","attrs":{"href":"https://kb.uconn.edu/space/SH/26033914267/Using+GUIs+on+the+HPC"}}]},{"text":". ","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Modules that I installed used to work properly on the original HPC but they are not loading properly on HPC 2.0. How do I resolve this problem? ","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"There are many reasons this might be occurring, but a common problem with user-installed programs is that the module names and versions have changed slightly between the old HPC and HPC 2.0. It may be that the dependencies your program used to rely on are no longer available on HPC 2.0. For instance, the GCC compilers have been updated and some of the old ones are no longer available. In this case, the ideal situation would be to install your program again using the newer compilers---this is often a good idea anyway because newer compilers sometimes increase the performance and reduce the chance of bugs. ","type":"text"}]},{"type":"paragraph","content":[{"text":"If there are extenuating circumstances that prevent you from using a program with new compilers or are experiencing other module-related problems, we invite you to submit a request for assistance by emailing ","type":"text"},{"text":"hpc@uconn.edu","type":"text","marks":[{"type":"link","attrs":{"href":"mailto:hpc@uconn.edu"}}]},{"text":". Then we can discuss options of how to set up a module that meets your needs.","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"How do I fix an error that says one module conflicts with other modules(s)?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"If the 'module load' command returns the following errors:","type":"text"}]},{"type":"codeBlock","content":[{"text":"$ module load \n(4):ERROR:150: Module '' conflicts with the currently loaded module(s) ''\n(4):ERROR:102: Tcl command execution failed: conflict \n","type":"text"}]},{"type":"paragraph","content":[{"text":"This means that the module you want to load conflicts with the currently loaded module, . To fix it, please unload and then load again:","type":"text"}]},{"type":"codeBlock","content":[{"text":"$ module unload \n$ module load \n","type":"text"}]},{"type":"paragraph","content":[{"text":"Or","type":"text"}]},{"type":"codeBlock","content":[{"text":"$ module switch \n","type":"text"}]},{"type":"paragraph","content":[{"text":"Or, if neither of these works, you can purge all the modules with","type":"text"}]},{"type":"codeBlock","content":[{"text":"$ module purge\n","type":"text"}]},{"type":"paragraph","content":[{"text":"and start fresh.","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"How do I fix an error that says the module I’m trying to load depends on other modules?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"If the 'module load' command returns the following errors:","type":"text"}]},{"type":"codeBlock","content":[{"text":"$ module load \n(9):ERROR:151: Module '' depends on one of the module(s) ''\n(9):ERROR:102: Tcl command execution failed: prereq \n","type":"text"}]},{"type":"paragraph","content":[{"text":"This means that the module you want to load depends on the module . To fix it, please load prior to :","type":"text"}]},{"type":"codeBlock","content":[{"text":"$ module load \n","type":"text"}]},{"type":"paragraph","content":[{"text":"You may encounter the above errors ","type":"text"},{"text":"many times","type":"text","marks":[{"type":"underline"}]},{"text":". Please load/unload the requested/conflicted modules and try again.","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Why do I get syntax errors when I try to run scripts via ","type":"text"},{"text":"sbatch","type":"text","marks":[{"type":"strong"}]},{"text":"?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"If the script could not run via ","type":"text"},{"text":"sbatch","type":"text","marks":[{"type":"strong"}]},{"text":". The errors usually look like this:","type":"text"}]},{"type":"codeBlock","content":[{"text":"sh: -c: line 0: unexpected EOF while looking for matching `\"'\nsh: -c: line 1: syntax error: unexpected end of file\n","type":"text"}]},{"type":"paragraph","content":[{"text":"It is usually due to the wrong file format. Your file is still in the Windows format but not the Linux format.","type":"text"}]},{"type":"codeBlock","content":[{"text":"$ file comsol.sh # with wrong format\ncomsol.sh: Little-endian UTF-16 Unicode English text, with CRLF line terminators\n$ iconv -f utf-16 -t ascii comsol.sh -o comsol.sh # Convert to ASCII first.\n$ dos2unix comsol.sh # change CRLF line terminators to Unix format\n$ file comsol.sh\ncomsol.sh: Bourne-Again shell script text executable\n","type":"text"}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":1},"content":[{"text":"Guidelines for Citing the HPC","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"How can I acknowledge the Storrs HPC in our publications?","type":"text"}]},{"type":"expand","attrs":{"title":""},"content":[{"type":"paragraph","content":[{"text":"When acknowledging the Storrs HPC cluster in your publications, you can use the following suggested text:","type":"text"}]},{"type":"paragraph","content":[{"text":"\"The computational work for this project was conducted using resources provided by the Storrs High-Performance Computing (HPC) cluster. We extend our gratitude to the UConn Storrs HPC and its team for their resources and support, which aided in achieving these results.\"","type":"text","marks":[{"type":"em"}]}]},{"type":"paragraph","content":[{"text":"For a more detailed acknowledgment, consider including specific information about the resources used, such as:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"The types of compute nodes utilized","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Storage resources accessed","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Software applications or libraries supported by the HPC","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Including these details can provide a clearer picture of the HPC resources that contributed to your research.","type":"text"}]}]}],"version":1}