Don't fail when the log directory already exists

dask_jobqueue/core.py

@@ -383,8 +383,7 @@ def __init__(
 
         self.log_directory = log_directory
         if self.log_directory is not None:
-            if not os.path.exists(self.log_directory):
-                os.makedirs(self.log_directory)
+            os.makedirs(self.log_directory, exist_ok=True)
 
     @classmethod
     def default_config_name(cls):

dask_jobqueue/tests/test_jobqueue_core.py

@@ -172,6 +172,21 @@ def test_log_directory(Cluster, tmpdir):
         assert os.path.exists(tmpdir.strpath)
 
 
+def test_log_directory_already_created(Cluster, tmpdir, monkeypatch):
+    # Another process can create the directory between a check for its
+    # existence and the creation itself, make sure this is not an error
+    real_makedirs = os.makedirs
+
+    def makedirs_after_race(name, *args, **kwargs):
+        real_makedirs(name, exist_ok=True)
+        return real_makedirs(name, *args, **kwargs)
+
+    log_directory = os.path.join(tmpdir.strpath, "logs")
+    monkeypatch.setattr(os, "makedirs", makedirs_after_race)
+    with Cluster(cores=1, memory="1GB", log_directory=log_directory):
+        assert os.path.isdir(log_directory)
+
+
 def test_cluster_has_cores_and_memory(Cluster):
     base_regex = r"{}.+".format(Cluster.__name__)
     with pytest.raises(ValueError, match=base_regex + r"cores=\d, memory='\d+GB'"):

docs/source/changelog.rst

@@ -4,6 +4,8 @@ Changelog
 Development version
 -------------------
 
+- Fixed a ``FileExistsError`` when several clusters create the same ``log_directory`` at the same time (:issue:`692`)
+
 0.9.0 / 2024-08-22
 ------------------