Stop running StageMWM when any country was failed.

Signed-off-by: Viktor Govako <viktor.govako@gmail.com>
This commit is contained in:
Viktor Govako
2022-04-17 14:58:49 +03:00
committed by Konstantin Pastbin
parent e298da740e
commit a576f529a4

View File

@@ -12,7 +12,7 @@ import os
import shutil import shutil
import tarfile import tarfile
from collections import defaultdict from collections import defaultdict
from multiprocessing.pool import ThreadPool from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import AnyStr from typing import AnyStr
from typing import Type from typing import Type
@@ -173,11 +173,10 @@ class StageMwm(Stage):
tmp_mwm_names = env.get_tmp_mwm_names() tmp_mwm_names = env.get_tmp_mwm_names()
if len(tmp_mwm_names): if len(tmp_mwm_names):
logger.info(f'Number of feature data .mwm.tmp country files to process: {len(tmp_mwm_names)}') logger.info(f'Number of feature data .mwm.tmp country files to process: {len(tmp_mwm_names)}')
with ThreadPool(settings.THREADS_COUNT) as pool: with ThreadPoolExecutor(settings.THREADS_COUNT) as pool:
pool.map( pool.map(
lambda c: StageMwm.make_mwm(c, env), lambda c: StageMwm.make_mwm(c, env),
tmp_mwm_names, tmp_mwm_names
chunksize=1,
) )
else: else:
# TODO: list all countries that were not found? # TODO: list all countries that were not found?