collect.py

commits = self.iter_commits()

jobs = config["core"].get("jobs")
max_workers = jobs if len(commits) > jobs else len(commits)
index_max_workers = None if jobs / max_workers < 2 else int(jobs / max_workers)

executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="dvc_collect")

def gen():
    for commit in commits:
        fs = fs_from_commit(commit)
        index = Index(fs, max_threads=index_max_workers)
        repo_fs = RepoFS(index, fs)
        yield repo_fs

with executor:
    results = list(executor.map(collect, gen()))