drifty/comaps
1
0
Fork 0
mirror of https://codeberg.org/comaps/comaps synced 2025-12-19 13:03:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Enable StageDescriptions.

Browse Source 
Signed-off-by: Viktor Govako <viktor.govako@gmail.com>
This commit is contained in:
Viktor Govako
2022-04-28 11:49:36 +03:00
committed by Konstantin Pastbin
parent a3bb0bed46
commit 68711cae06
2 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tools/python/descriptions/descriptions_downloader.py
View File

@@ -7,6 +7,8 @@ import time
import types import types
import urllib.error import urllib.error
import urllib.parse import urllib.parse
import http.client
from multiprocessing.pool import ThreadPool from multiprocessing.pool import ThreadPool
import htmlmin import htmlmin
@@ -73,6 +75,7 @@ def try_get(obj, prop, *args, **kwargs):
requests.exceptions.ConnectionError, requests.exceptions.ConnectionError,
requests.exceptions.ReadTimeout, requests.exceptions.ReadTimeout,
json.decoder.JSONDecodeError, json.decoder.JSONDecodeError,
http.client.HTTPException,
): ):
time.sleep(random.uniform(0.0, 1.0 / 1000.0 * ATTEMPTS_PAUSE_MS)) time.sleep(random.uniform(0.0, 1.0 / 1000.0 * ATTEMPTS_PAUSE_MS))
attempts -= 1 attempts -= 1
@@ -236,7 +239,7 @@ def wikipedia_worker(output_dir, checker, langs):
if not checker(ident): if not checker(ident):
return return
url = url.strip() url = url.strip()
except (AttributeError, IndexError): except (AttributeError, ValueError):
log.exception(f"{line} is incorrect.") log.exception(f"{line} is incorrect.")
return return
parsed = urllib.parse.urlparse(url) parsed = urllib.parse.urlparse(url)
@@ -279,7 +282,7 @@ def wikidata_worker(output_dir, checker, langs):
wikidata_id = wikidata_id.strip() wikidata_id = wikidata_id.strip()
if not checker(ident): if not checker(ident):
return return
except (AttributeError, IndexError): except (AttributeError, ValueError):
log.exception(f"{line} is incorrect.") log.exception(f"{line} is incorrect.")
return return
client = Client() client = Client()

2
tools/python/maps_generator/generator/stages_declaration.py
View File

@@ -140,7 +140,6 @@ class StageFeatures(Stage):
@outer_stage @outer_stage
@production_only
@helper_stage_for("StageDescriptions") @helper_stage_for("StageDescriptions")
class StageDownloadDescriptions(Stage): class StageDownloadDescriptions(Stage):
def apply(self, env: Env): def apply(self, env: Env):
@@ -292,7 +291,6 @@ class StageIsolinesInfo(Stage):
@country_stage @country_stage
@production_only
class StageDescriptions(Stage): class StageDescriptions(Stage):
def apply(self, env: Env, country, **kwargs): def apply(self, env: Env, country, **kwargs):
steps.step_description(env, country, **kwargs) steps.step_description(env, country, **kwargs)