flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/activities/social_services/measurements/collect.py", line 311, in flow
DF.dump_to_sql(dict(
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/to_sql.py", line 67, in __init__
with self.engine.connect():
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3295, in connect
return self._connection_cls(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 146, in __init__
Connection._handle_dbapi_exception_noconnection(
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2450, in _handle_dbapi_exception_noconnection
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
(Background on this error at: https://sqlalche.me/e/20/e3q8)
Pipeline ID: activities/social_services/measurements/collect
Pipeline ID: activities/social_services/social_service_reports
Pipeline ID: activities/social_services/social_service_suppliers
Pipeline ID: activities/social_services/social_service_tenders
Pipeline ID: activities/social_services/social_services
scraper
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/connection.py", line 95, in create_connection
raise err
File "/usr/local/lib/python3.9/site-packages/urllib3/util/connection.py", line 85, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 716, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 416, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 244, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/usr/local/lib/python3.9/http/client.py", line 1285, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/lib/python3.9/http/client.py", line 1331, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.9/http/client.py", line 1280, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.9/http/client.py", line 1040, in _send_output
self.send(msg)
File "/usr/local/lib/python3.9/http/client.py", line 980, in send
self.connect()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 205, in connect
conn = self._new_conn()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 186, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7fb30de60580>: Failed to establish a new connection: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 644, in send
resp = conn.urlopen(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 802, in urlopen
retries = retries.increment(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/retry.py", line 594, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='www.boi.org.il', port=80): Max retries exceeded with url: /he/DataAndStatistics/_layouts/boi/handlers/WebPartHandler.aspx?wp=SeriesSearchBySubject&lang=he-IL&PaggingSize=100&Level=4&sId=1 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fb30de60580>: Failed to establish a new connection: [Errno 111] Connection refused'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/pipelines/bank_of_israel/scraper.py", line 124, in <module>
series_sum = get_all_series()
File "/datapackage_pipelines_budgetkey/pipelines/bank_of_israel/scraper.py", line 23, in get_all_series
series = get_series(sid)
File "/datapackage_pipelines_budgetkey/pipelines/bank_of_israel/scraper.py", line 34, in get_series
response = session.get(SEARCH_PAGE_RESULTS_URL.format(sid=sid))
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 677, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='www.boi.org.il', port=80): Max retries exceeded with url: /he/DataAndStatistics/_layouts/boi/handlers/WebPartHandler.aspx?wp=SeriesSearchBySubject&lang=he-IL&PaggingSize=100&Level=4&sId=1 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fb30de60580>: Failed to establish a new connection: [Errno 111] Connection refused'))
Pipeline ID: bank_of_israel/bank_of_israel
flow
ERROR :/usr/local/lib/python3.9/site-packages/dataflows/processors/join.py:189: UserWarning: For the `join` processor the `full=True` flag is deprecated. Please use the "mode" parameter instead.
warnings.warn(
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/budget/municipal/datacity-budget-fetcher.py", line 177, in flow
DF.dump_to_sql(dict(
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/to_sql.py", line 67, in __init__
with self.engine.connect():
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3295, in connect
return self._connection_cls(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 146, in __init__
Connection._handle_dbapi_exception_noconnection(
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2450, in _handle_dbapi_exception_noconnection
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
(Background on this error at: https://sqlalche.me/e/20/e3q8)
Pipeline ID: budget/municipal/fetcher
Pipeline ID: budget/national/changes/explanations/all
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/budget/national/changes/original/committee-zipfile.py", line 14, in flow
archive_ = gcl.download(WEIRD_ZIP_FILE, outfile=f'p{YEAR}.zip')
File "/datapackage_pipelines_budgetkey/common/google_chrome.py", line 219, in download
assert False, 'Failed to download file, %r' % downloads
AssertionError: Failed to download file, ['']
DEBUG :[chan 14] Max packet in: 32768 bytes
DEBUG :[chan 14] Max packet out: 32768 bytes
DEBUG :Secsh channel 14 opened.
DEBUG :[chan 14] Sesch channel 14 request ok
DEBUG :[chan 14] EOF received (14)
DEBUG :[chan 14] EOF sent (14)
DEBUG :Dropping user packet because connection is dead.
Pipeline ID: budget/national/changes/original/committee-zipfile
Pipeline ID: budget/national/changes/original/current-year-fixes
Pipeline ID: budget/national/changes/original/national-budget-changes
Pipeline ID: budget/national/changes/processed/national-budget-changes-aggregated
Pipeline ID: budget/national/changes/processed/transactions
We get the data the Ministry of Finance publishes in data.gov.il every year. It comes in XLS format, with one row per TAKANA and phase (original, approved, executed). In this pipeline we create from each triplet a single row that has all the data.
Pipeline ID: budget/national/original/national-budgets
This pipeline joins the different phases of the budget (allocated, revised and executed). In the original file there's a separate row for each of the phases. We like it better as a single row with all phase info. Another thing this pipeline does is to rename the column titles - to more friendly English names. Also, it create rows for all hierarchies - where upper hierarchies (2, 4 & 6 digits) are plain aggregations of the 8-digit items they contain.
Pipeline ID: budget/national/processed/aggregated-yearly
Pipeline ID: budget/national/processed/category-explanations
This pipeline joins budget items that span across years.
Pipeline ID: budget/national/processed/connected-items-explained
This pipeline joins budget items that span across years.
Pipeline ID: budget/national/processed/connected-national-budgets
Pipeline ID: budget/national/processed/just-the-total
Pipeline ID: budget/national/processed/roof-names
This pipeline joins the budget data to itself so that each item has a list of its immediate children.
Pipeline ID: budget/national/processed/with-extras
Pipeline ID: budgetkey/analysis/spending/publisher_entity_analysis
Pipeline ID: budgetkey/analysis/spending/publisher_foa_analysis
Pipeline ID: budgetkey/analysis/spending/united
Pipeline ID: budgetkey/budget-functional-aggregates
Pipeline ID: budgetkey/documents
Pipeline ID: budgetkey/elasticsearch/index_activities
Pipeline ID: budgetkey/elasticsearch/index_budget
Invalid dependency :Cannot run until dependency passes validation: ./procurement/calls_for_bids/calls-for-bids-all From ./procurement/calls_for_bids/calls-for-bids-all :['Dependency unsuccessful', 'Cannot run until dependency "./procurement/calls_for_bids/gov-publications" is successfully executed'] From ./procurement/calls_for_bids/calls-for-bids-all :['Dependency unsuccessful', 'Cannot run until dependency "./procurement/calls_for_bids/gov-publications2" is successfully executed'] From ./procurement/calls_for_bids/calls-for-bids-all :['Dependency unsuccessful', 'Cannot run until dependency "./procurement/calls_for_bids/gov-publications-ministry-of-immigration" is successfully executed']
Pipeline ID: budgetkey/elasticsearch/index_calls_for_bids
Pipeline ID: budgetkey/elasticsearch/index_contract_spending
dump_to_es
ERROR :DUMP TO ES ERROR BadRequestError(400, 'illegal_argument_exception', 'mapper [details.pob] cannot be changed from type [text] to [long]')
ERROR :TB
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/dump_to_es.py", line 163, in <module>
spew_flow(flow(ctx.parameters), ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__
spew(self.datapackage, self.resource_iterator, stats=self.stats)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 68, in spew
for res in resources_iterator:
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 68, in <genexpr>
res_iter = (it if isinstance(it, ResourceWrapper) else ResourceWrapper(res, it)
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/dumper_base.py", line 82, in process_resources
ret = self.process_resource(
File "/usr/local/lib/python3.9/site-packages/dataflows_elasticsearch/processors/dump_to_elasticsearch.py", line 69, in process_resource
storage.create(index_name, res.descriptor['schema'],
File "/usr/local/lib/python3.9/site-packages/tableschema_elasticsearch/storage.py", line 123, in create
self.put_mapping(bucket, descriptor, index_name, mapping_generator_cls)
File "/usr/local/lib/python3.9/site-packages/tableschema_elasticsearch/storage.py", line 85, in put_mapping
self.__es.indices.put_mapping(index=index_name, **mapping)
File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/utils.py", line 458, in wrapped
return api(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/indices.py", line 4137, in put_mapping
return self.perform_request( # type: ignore[return-value]
File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/_base.py", line 422, in perform_request
return self._client.perform_request(
File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/_base.py", line 271, in perform_request
response = self._perform_request(
File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/_base.py", line 351, in _perform_request
raise HTTP_EXCEPTIONS.get(meta.status, ApiError)(
elasticsearch.BadRequestError: BadRequestError(400, 'illegal_argument_exception', 'mapper [details.pob] cannot be changed from type [text] to [long]')
ERROR :Traceback (most recent call last):
ERROR :File "/datapackage_pipelines_budgetkey/processors/dump_to_es.py", line 163, in <module>
ERROR :spew_flow(flow(ctx.parameters), ctx)
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__
ERROR :spew(self.datapackage, self.resource_iterator, stats=self.stats)
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 68, in spew
ERROR :for res in resources_iterator:
ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 68, in <genexpr>
ERROR :res_iter = (it if isinstance(it, ResourceWrapper) else ResourceWrapper(res, it)
ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/dumper_base.py", line 82, in process_resources
ERROR :ret = self.process_resource(
ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows_elasticsearch/processors/dump_to_elasticsearch.py", line 69, in process_resource
ERROR :storage.create(index_name, res.descriptor['schema'],
ERROR :File "/usr/local/lib/python3.9/site-packages/tableschema_elasticsearch/storage.py", line 123, in create
ERROR :self.put_mapping(bucket, descriptor, index_name, mapping_generator_cls)
ERROR :File "/usr/local/lib/python3.9/site-packages/tableschema_elasticsearch/storage.py", line 85, in put_mapping
ERROR :self.__es.indices.put_mapping(index=index_name, **mapping)
ERROR :File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/utils.py", line 458, in wrapped
ERROR :return api(*args, **kwargs)
ERROR :File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/indices.py", line 4137, in put_mapping
ERROR :return self.perform_request( # type: ignore[return-value]
ERROR :File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/_base.py", line 422, in perform_request
ERROR :return self._client.perform_request(
ERROR :File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/_base.py", line 271, in perform_request
ERROR :response = self._perform_request(
ERROR :File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/_base.py", line 351, in _perform_request
ERROR :raise HTTP_EXCEPTIONS.get(meta.status, ApiError)(
ERROR :elasticsearch
ERROR :.
ERROR :BadRequestError
ERROR ::
ERROR :BadRequestError(400, 'illegal_argument_exception', 'mapper [details.pob] cannot be changed from type [text] to [long]')
Pipeline ID: budgetkey/elasticsearch/index_entities
Pipeline ID: budgetkey/elasticsearch/index_gov_decisions
Pipeline ID: budgetkey/elasticsearch/index_muni_budgets
Pipeline ID: budgetkey/elasticsearch/index_muni_tenders
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/openai/_base_client.py", line 1113, in request
response.raise_for_status()
File "/usr/local/lib/python3.9/site-packages/httpx/_models.py", line 829, in raise_for_status
raise HTTPStatusError(message, request=request, response=self)
httpx.HTTPStatusError: Client error '400 Bad Request' for url 'https://api.openai.com/v1/embeddings'
For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/400
DEBUG :Not retrying
DEBUG :Re-raising status error
ERROR :Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module>
spew_flow(flow, ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__
spew(self.datapackage, self.resource_iterator, stats=self.stats)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew
for rec in res:
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/rows_processor.py", line 11, in process_resource
yield from self.func(resource)
File "/datapackage_pipelines_budgetkey/pipelines/budgetkey/elasticsearch/chunker.py", line 47, in func
hit, embedding = embed(chunk)
File "/datapackage_pipelines_budgetkey/common/cached_openai.py", line 42, in embed
embedding = client.embeddings.create(
File "/usr/local/lib/python3.9/site-packages/openai/resources/embeddings.py", line 136, in create
return self._post(
File "/usr/local/lib/python3.9/site-packages/openai/_base_client.py", line 1360, in post
return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))
File "/usr/local/lib/python3.9/site-packages/openai/_base_client.py", line 1133, in request
raise self._make_status_error_from_response(err.response) from None
openai.BadRequestError: Error code: 400 - {'error': {'message': "Invalid 'input': maximum context length is 8192 tokens.", 'type': 'invalid_request_error', 'param': None, 'code': None}}
DEBUG :close.started
DEBUG :close.complete
Pipeline ID: budgetkey/elasticsearch/index_national_budget_changes
Invalid dependency :Cannot run until dependency passes validation: ./people/aggregated From ./people/aggregated :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/all'] From ./people/aggregated :['From ./people/all', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/all-appointments']] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/media/themarker/themarker" is successfully executed']]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-nominations']]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-nominations', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list']]]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-nominations', ['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']]]]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-nominations', ['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]]]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-resignations']]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-resignations', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list']]]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-resignations', ['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']]]]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-resignations', ['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]]]] From ./people/aggregated :['From ./people/all', ['Dependency unsuccessful', 'Cannot run until dependency "./people/political_candidates/election_candidates21" is successfully executed']]
Pipeline ID: budgetkey/elasticsearch/index_people
Pipeline ID: budgetkey/elasticsearch/index_reports
Dependency unsuccessful :Cannot run until dependency "./supports/criteria/support-criteria" is successfully executed
Pipeline ID: budgetkey/elasticsearch/index_support_criteria
Pipeline ID: budgetkey/elasticsearch/index_support_programs
Pipeline ID: budgetkey/elasticsearch/index_supports
Pipeline ID: budgetkey/elasticsearch/index_tenders
Pipeline ID: budgetkey/elasticsearch/index_units
Dependency unsuccessful :Cannot run until dependency "./budgetkey/elasticsearch/index_national_budget_changes" is successfully executed Dependency unsuccessful :Cannot run until dependency "./budgetkey/elasticsearch/index_entities" is successfully executed
Pipeline ID: budgetkey/elasticsearch/sitemaps
Pipeline ID: budgetkey/emails/emails
Pipeline ID: donations/candidates
Pipeline ID: donations/parties
get_transactions
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/pipelines/donations/get_transactions.py", line 150, in <module>
spew(dp, process_resources(res_iter))
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew
for rec in res:
File "/datapackage_pipelines_budgetkey/pipelines/donations/get_transactions.py", line 109, in get_transactions
for resp in self.get_for_candidate(cid):
File "/datapackage_pipelines_budgetkey/pipelines/donations/get_transactions.py", line 95, in get_for_candidate
resp = self.get_for_range(cid, range_start, range_end)
File "/datapackage_pipelines_budgetkey/pipelines/donations/get_transactions.py", line 70, in get_for_range
assert len(resp) == 6
TypeError: object of type 'NoneType' has no len()
Pipeline ID: donations/transactions
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/education/tochniyot/scraper.py", line 103, in flow
return Flow(scrape(),
File "/datapackage_pipelines_budgetkey/pipelines/education/tochniyot/scraper.py", line 65, in scrape
education_programs_count = get_education_programs_count()
File "/datapackage_pipelines_budgetkey/pipelines/education/tochniyot/scraper.py", line 56, in get_education_programs_count
res = send_tochniyot_request()
File "/datapackage_pipelines_budgetkey/pipelines/education/tochniyot/scraper.py", line 80, in send_tochniyot_request
resp.raise_for_status()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 1026, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 500 Server Error: Internal Server Error for url: https://apps.education.gov.il/TyhNet/ClientWs/TochnitCh.asmx/IturTochnitChByMeafyenim
Pipeline ID: education/tochniyot/tochniyot
Dependency unsuccessful :Cannot run until dependency "./entities/special/registry" is successfully executed
Pipeline ID: entities/all
Pipeline ID: entities/associations/guidestar/guidestar
Pipeline ID: entities/associations/guidestar/guidestar-scraper
רשימת כלל העמותות הפעילות, מתעדכן שבועית.
Pipeline ID: entities/associations/registrar/registry
Pipeline ID: entities/companies/registrar/details
רשימת כלל החברות הפעילות, מתעדכן שבועית.
dump.to_sql
ERROR :Traceback (most recent call last):
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2127, in _exec_insertmany_context
ERROR :dialect.do_execute(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 952, in do_execute
ERROR :cursor.execute(statement, parameters)
ERROR :psycopg2
ERROR :.
ERROR :OperationalError
ERROR ::
ERROR :server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
ERROR :The above exception was the direct cause of the following exception:
ERROR :Traceback (most recent call last):
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/dump/to_sql.py", line 15, in <module>
ERROR :spew_flow(flow(ctx.parameters), ctx)
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__
ERROR :spew(self.datapackage, self.resource_iterator, stats=self.stats)
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew
ERROR :for rec in res:
ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/dumper_base.py", line 69, in row_counter
ERROR :for row in iterator:
ERROR :File "/usr/local/lib/python3.9/site-packages/tableschema_sql/writer.py", line 55, in write
ERROR :for wr in self.__insert(connection):
ERROR :File "/usr/local/lib/python3.9/site-packages/tableschema_sql/writer.py", line 86, in __insert
ERROR :connection.execute(statement, self.__buffer)
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1421, in execute
ERROR :return meth(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 526, in _execute_on_connection
ERROR :return connection._execute_clauseelement(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1643, in _execute_clauseelement
ERROR :ret = self._execute_context(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1846, in _execute_context
ERROR :return self._exec_insertmany_context(dialect, context)
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2135, in _exec_insertmany_context
ERROR :self._handle_dbapi_exception(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2365, in _handle_dbapi_exception
ERROR :raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2127, in _exec_insertmany_context
ERROR :dialect.do_execute(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 952, in do_execute
ERROR :cursor.execute(statement, parameters)
ERROR :sqlalchemy.exc
ERROR :.
ERROR :OperationalError
ERROR ::
ERROR :(psycopg2.OperationalError) server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
[SQL: INSERT INTO company_details (id, company_name, company_name_eng, company_type, company_status, company_description, company_goal, company_registration_date, company_is_government, company_limit, company_is_mafera, company_last_report_year, company_ci ... 534839 characters truncated ... pany_pob__999)s, %(company_country__999)s, %(company_located_at__999)s, %(company_sub_status__999)s)]
[parameters: {'company_located_at__0': None, 'company_pob__0': None, 'company_name__0': 'לונה נייל בע״מ', 'id__0': '512957408', 'company_postal_code__0': '6433812', 'company_description__0': None, 'company_country__0': 'ישראל', 'company_limit__0': 'מוגבלת', 'company_sub_status__0': None, 'company_registration_date__0': datetime.date(2000, 5, 23), 'company_name_eng__0': None, 'company_type__0': 'ישראלית חברה פרטית', 'company_last_report_year__0': None, 'company_street__0': "המלך ג'ורג'", 'company_status__0': 'פעילה', 'company_goal__0': 'לעסוק בכל עיסוק חוקי', 'company_is_government__0': False, 'company_is_mafera__0': True, 'company_street_number__0': '88', 'company_city__0': 'תל אביב - יפו', 'company_located_at__1': 'מגדל לוינשטיין קומה 29', 'company_pob__1': None, 'company_name__1': 'טק-איז (ישראל) בע״מ', 'id__1': '512957416', 'company_postal_code__1': '66182', 'company_description__1': None, 'company_country__1': 'ישראל', 'company_limit__1': 'מוגבלת', 'company_sub_status__1': None, 'company_registration_date__1': datetime.date(2000, 5, 23), 'company_name_eng__1': 'TECC-IS (ISRAEL) LTD', 'company_type__1': 'ישראלית חברה פרטית', 'company_last_report_year__1': 2003, 'company_street__1': 'מנחם בגין', 'company_status__1': 'מחוסלת מרצון', 'company_goal__1': 'לעסוק בכל עיסוק חוקי', 'company_is_government__1': False, 'company_is_mafera__1': None, 'company_street_number__1': '23', 'company_city__1': 'תל אביב - יפו', 'company_located_at__2': 'א.ת צפוני לוד', 'company_pob__2': None, 'company_name__2': 'קשת יהודה עבודות בניין ואחזקות בע״מ', 'id__2': '512957424', 'company_postal_code__2': '7120517', 'company_description__2': None, 'company_country__2': 'ישראל', 'company_limit__2': 'מוגבלת', 'company_sub_status__2': None, 'company_registration_date__2': datetime.date(2000, 5, 23) ... 19900 parameters truncated ... 'company_name_eng__997': None, 'company_type__997': 'ישראלית חברה פרטית', 'company_last_report_year__997': None, 'company_street__997': 'אייזק כהן', 'company_status__997': 'מחוסלת מרצון', 'company_goal__997': 'לעסוק בכל עיסוק חוקי', 'company_is_government__997': False, 'company_is_mafera__997': None, 'company_street_number__997': '275', 'company_city__997': 'צפת', 'company_located_at__998': None, 'company_pob__998': None, 'company_name__998': 'שער הטכניון חברה לניהול בע״מ', 'id__998': '512967712', 'company_postal_code__998': '2751021', 'company_description__998': None, 'company_country__998': 'ישראל', 'company_limit__998': 'מוגבלת', 'company_sub_status__998': None, 'company_registration_date__998': datetime.date(2000, 6, 15), 'company_name_eng__998': None, 'company_type__998': 'ישראלית חברה פרטית', 'company_last_report_year__998': 2025, 'company_street__998': 'חרושת', 'company_status__998': 'פעילה', 'company_goal__998': 'לעסוק בכל עיסוק חוקי', 'company_is_government__998': False, 'company_is_mafera__998': None, 'company_street_number__998': '4', 'company_city__998': 'קריית ביאליק', 'company_located_at__999': None, 'company_pob__999': None, 'company_name__999': 'אורות מטבחים בע״מ', 'id__999': '512967720', 'company_postal_code__999': '5960602', 'company_description__999': None, 'company_country__999': 'ישראל', 'company_limit__999': 'מוגבלת', 'company_sub_status__999': None, 'company_registration_date__999': datetime.date(2000, 6, 15), 'company_name_eng__999': None, 'company_type__999': 'ישראלית חברה פרטית', 'company_last_report_year__999': 2004, 'company_street__999': 'הפלדה', 'company_status__999': 'פעילה', 'company_goal__999': 'לעסוק בכל עיסוק חוקי', 'company_is_government__999': False, 'company_is_mafera__999': True, 'company_street_number__999': '4', 'company_city__999': 'בת ים'}]
(Background on this error at: https://sqlalche.me/e/20/e3q8)
Pipeline ID: entities/companies/registrar/scraper
רשימת אגודות שיתופיות, מתוך ממשק נתונים של מאגר של משרד הכלכלה / האגף לאיגוד שיתופי, מתעדכן שבועית
Pipeline ID: entities/cooperatives/cooperatives
Pipeline ID: entities/endowments/endowments
Pipeline ID: entities/entities
Pipeline ID: entities/fingerprints
fingerprint
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/fingerprint.py", line 201, in <module>
resource = next(iter(filter(
StopIteration
Pipeline ID: entities/foi_orgs/foi_orgs
scraper
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/pipelines/entities/moital/scraper.py", line 79, in <module>
spew(datapackage, [scrape()])
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew
for rec in res:
File "/datapackage_pipelines_budgetkey/pipelines/entities/moital/scraper.py", line 32, in scrape
desired_capabilities=DesiredCapabilities.PHANTOMJS)
AttributeError: type object 'DesiredCapabilities' has no attribute 'PHANTOMJS'
Pipeline ID: entities/moital/moital_service_providers
רשימת אגודות עותומניות, מתוך תשובה לבקשת חופש מידע
Pipeline ID: entities/ottoman/ottoman-association-registry
Pipeline ID: entities/partnerships/partnerships
stream_remote_resources
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/connection.py", line 72, in create_connection
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
File "/usr/local/lib/python3.9/socket.py", line 966, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 716, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 416, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 244, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/usr/local/lib/python3.9/http/client.py", line 1285, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/lib/python3.9/http/client.py", line 1331, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.9/http/client.py", line 1280, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.9/http/client.py", line 1040, in _send_output
self.send(msg)
File "/usr/local/lib/python3.9/http/client.py", line 980, in send
self.connect()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 205, in connect
conn = self._new_conn()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 186, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7f7bf46da7f0>: Failed to establish a new connection: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 644, in send
resp = conn.urlopen(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 802, in urlopen
retries = retries.increment(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/retry.py", line 594, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='employment.molsa.gov.il', port=80): Max retries exceeded with url: /Employment/SafetyAndHealth/Enforcement/Documents/%D7%A6%D7%95%D7%95%D7%99%20%D7%91%D7%98%D7%99%D7%97%D7%95%D7%AA.xlsx (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f7bf46da7f0>: Failed to establish a new connection: [Errno -2] Name or service not known'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 63, in load
bytes = _RemoteStream(source, self.__http_session, self.__http_timeout).open()
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 121, in open
self.seek(0)
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 142, in seek
self.__response = self.__session.get(self.__source, stream=True, timeout=self.__timeout)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 677, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='employment.molsa.gov.il', port=80): Max retries exceeded with url: /Employment/SafetyAndHealth/Enforcement/Documents/%D7%A6%D7%95%D7%95%D7%99%20%D7%91%D7%98%D7%99%D7%97%D7%95%D7%AA.xlsx (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f7bf46da7f0>: Failed to establish a new connection: [Errno -2] Name or service not known'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/stream_remote_resources.py", line 244, in <module>
rows = stream_reader(resource, url, ignore_missing or url == "", limit_rows,
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/stream_remote_resources.py", line 199, in stream_reader
schema, headers, columns, stream, close = get_opener(url, _resource)()
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/stream_remote_resources.py", line 175, in opener
_stream.open()
File "/usr/local/lib/python3.9/site-packages/tabulator/stream.py", line 425, in open
self.__parser.open(source, encoding=self.__encoding)
File "/usr/local/lib/python3.9/site-packages/tabulator/parsers/xlsx.py", line 81, in open
source_bytes = self.__loader.load(source, mode="b", encoding=encoding)
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 72, in load
raise exceptions.HTTPError(str(exception))
tabulator.exceptions.HTTPError: HTTPConnectionPool(host='employment.molsa.gov.il', port=80): Max retries exceeded with url: /Employment/SafetyAndHealth/Enforcement/Documents/%D7%A6%D7%95%D7%95%D7%99%20%D7%91%D7%98%D7%99%D7%97%D7%95%D7%AA.xlsx (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f7bf46da7f0>: Failed to establish a new connection: [Errno -2] Name or service not known'))
Pipeline ID: entities/safety-orders/safety-orders
Dependency unsuccessful :Cannot run until dependency "./entities/safety-orders/safety-orders" is successfully executed
Pipeline ID: entities/safety-orders/safety-orders-by-entity
dump.to_sql
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/dump/to_sql.py", line 15, in <module>
spew_flow(flow(ctx.parameters), ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/lib/dump_to_sql.py", line 8, in flow
dump_to_sql(
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/to_sql.py", line 67, in __init__
with self.engine.connect():
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3295, in connect
return self._connection_cls(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 146, in __init__
Connection._handle_dbapi_exception_noconnection(
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2450, in _handle_dbapi_exception_noconnection
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
(Background on this error at: https://sqlalche.me/e/20/e3q8)
Pipeline ID: entities/special/registry
data_gov_il_resource
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 976, in json
return complexjson.loads(self.text, **kwargs)
File "/usr/local/lib/python3.9/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/local/lib/python3.9/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/local/lib/python3.9/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 23, in search_dataset
results = requests.get(PACKAGE_GET_URL + dataset_name, headers=HEADERS).json()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 980, in json
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 92, in get_resource
dataset = search_dataset(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 27, in search_dataset
results = gcd.json(PACKAGE_GET_URL + dataset_name)
AttributeError: 'function' object has no attribute 'json'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 40, in get_page
assert test(page)
UnboundLocalError: local variable 'page' referenced before assignment
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 66, in <module>
spew_flow(flow(ctx.parameters), ctx)
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 47, in flow
url, path = get_resource(gcd, dataset_name, resource_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 94, in get_resource
dataset = get_dataset_html(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 49, in get_dataset_html
page = get_page(gcd, PACKAGE_PAGE_URL + dataset_name, lambda page: 'resource-item' in page)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 42, in get_page
gcd.driver.get(url)
AttributeError: 'function' object has no attribute 'driver'
Pipeline ID: ezvonot/ezvonot_accounts
data_gov_il_resource
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 976, in json
return complexjson.loads(self.text, **kwargs)
File "/usr/local/lib/python3.9/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/local/lib/python3.9/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/local/lib/python3.9/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 23, in search_dataset
results = requests.get(PACKAGE_GET_URL + dataset_name, headers=HEADERS).json()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 980, in json
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 92, in get_resource
dataset = search_dataset(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 27, in search_dataset
results = gcd.json(PACKAGE_GET_URL + dataset_name)
AttributeError: 'function' object has no attribute 'json'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 40, in get_page
assert test(page)
UnboundLocalError: local variable 'page' referenced before assignment
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 66, in <module>
spew_flow(flow(ctx.parameters), ctx)
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 47, in flow
url, path = get_resource(gcd, dataset_name, resource_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 94, in get_resource
dataset = get_dataset_html(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 49, in get_dataset_html
page = get_page(gcd, PACKAGE_PAGE_URL + dataset_name, lambda page: 'resource-item' in page)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 42, in get_page
gcd.driver.get(url)
AttributeError: 'function' object has no attribute 'driver'
Pipeline ID: ezvonot/ezvonot_cases
Pipeline ID: ezvonot/ezvonot_crontab
data_gov_il_resource
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 976, in json
return complexjson.loads(self.text, **kwargs)
File "/usr/local/lib/python3.9/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/local/lib/python3.9/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/local/lib/python3.9/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 23, in search_dataset
results = requests.get(PACKAGE_GET_URL + dataset_name, headers=HEADERS).json()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 980, in json
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 92, in get_resource
dataset = search_dataset(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 27, in search_dataset
results = gcd.json(PACKAGE_GET_URL + dataset_name)
AttributeError: 'function' object has no attribute 'json'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 40, in get_page
assert test(page)
UnboundLocalError: local variable 'page' referenced before assignment
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 66, in <module>
spew_flow(flow(ctx.parameters), ctx)
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 47, in flow
url, path = get_resource(gcd, dataset_name, resource_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 94, in get_resource
dataset = get_dataset_html(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 49, in get_dataset_html
page = get_page(gcd, PACKAGE_PAGE_URL + dataset_name, lambda page: 'resource-item' in page)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 42, in get_page
gcd.driver.get(url)
AttributeError: 'function' object has no attribute 'driver'
Pipeline ID: ezvonot/ezvonot_decisions
Dependency unsuccessful :Cannot run until dependency "./ezvonot/ezvonot_accounts" is successfully executed Dependency unsuccessful :Cannot run until dependency "./ezvonot/ezvonot_cases" is successfully executed Dependency unsuccessful :Cannot run until dependency "./ezvonot/ezvonot_purposes" is successfully executed Dependency unsuccessful :Cannot run until dependency "./ezvonot/ezvonot_decisions" is successfully executed
Pipeline ID: ezvonot/ezvonot_decisions_join
data_gov_il_resource
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 976, in json
return complexjson.loads(self.text, **kwargs)
File "/usr/local/lib/python3.9/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/local/lib/python3.9/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/local/lib/python3.9/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 23, in search_dataset
results = requests.get(PACKAGE_GET_URL + dataset_name, headers=HEADERS).json()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 980, in json
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 92, in get_resource
dataset = search_dataset(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 27, in search_dataset
results = gcd.json(PACKAGE_GET_URL + dataset_name)
AttributeError: 'function' object has no attribute 'json'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 40, in get_page
assert test(page)
UnboundLocalError: local variable 'page' referenced before assignment
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 66, in <module>
spew_flow(flow(ctx.parameters), ctx)
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 47, in flow
url, path = get_resource(gcd, dataset_name, resource_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 94, in get_resource
dataset = get_dataset_html(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 49, in get_dataset_html
page = get_page(gcd, PACKAGE_PAGE_URL + dataset_name, lambda page: 'resource-item' in page)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 42, in get_page
gcd.driver.get(url)
AttributeError: 'function' object has no attribute 'driver'
Pipeline ID: ezvonot/ezvonot_deposits
data_gov_il_resource
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 976, in json
return complexjson.loads(self.text, **kwargs)
File "/usr/local/lib/python3.9/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/local/lib/python3.9/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/local/lib/python3.9/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 23, in search_dataset
results = requests.get(PACKAGE_GET_URL + dataset_name, headers=HEADERS).json()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 980, in json
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 92, in get_resource
dataset = search_dataset(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 27, in search_dataset
results = gcd.json(PACKAGE_GET_URL + dataset_name)
AttributeError: 'function' object has no attribute 'json'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 40, in get_page
assert test(page)
UnboundLocalError: local variable 'page' referenced before assignment
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 66, in <module>
spew_flow(flow(ctx.parameters), ctx)
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 47, in flow
url, path = get_resource(gcd, dataset_name, resource_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 94, in get_resource
dataset = get_dataset_html(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 49, in get_dataset_html
page = get_page(gcd, PACKAGE_PAGE_URL + dataset_name, lambda page: 'resource-item' in page)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 42, in get_page
gcd.driver.get(url)
AttributeError: 'function' object has no attribute 'driver'
Pipeline ID: ezvonot/ezvonot_purposes
Dependency unsuccessful :Cannot run until dependency "./facilities/labor/facilities_labor" is successfully executed
Pipeline ID: facilities/all/facilities_all
Pipeline ID: facilities/all/facilities_tamat
Pipeline ID: facilities/education/facilities_education
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/facilities/labor/mol.py", line 77, in flow
DF.dump_to_sql(dict(
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/to_sql.py", line 67, in __init__
with self.engine.connect():
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3295, in connect
return self._connection_cls(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 146, in __init__
Connection._handle_dbapi_exception_noconnection(
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2450, in _handle_dbapi_exception_noconnection
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
(Background on this error at: https://sqlalche.me/e/20/e3q8)
Pipeline ID: facilities/labor/facilities_labor
Pipeline ID: facilities/mapathataf-export/mapathataf_export
Pipeline ID: facilities/tipat-halav/facilities_tipat_halav
Pipeline ID: facilities/welfare/facilities_welfare
Pipeline ID: government_decisions/government_decisions
Pipeline ID: government_decisions/scraper
(sink)
ERROR :Data Package validation error: Descriptor validation error: [] should be non-empty at "schema/fields" in descriptor and at "properties/schema/properties/fields/minItems" in profile
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/manager/../lib/internal/sink.py", line 24, in <module>
params, dp, res_iter = ingest()
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 186, in ingest
params, datapackage, resource_iterator = _ingest(debug=debug)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 44, in _ingest
datapackage, resource_iterator, dependency_dp = process_input(sys.stdin, validate, debug)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/input_processor.py", line 88, in process_input
datapackage.validate(dp_to_validate)
File "/usr/local/lib/python3.9/site-packages/datapackage/validate.py", line 28, in validate
Package(descriptor, strict=True)
File "/usr/local/lib/python3.9/site-packages/datapackage/package.py", line 126, in __init__
self.__build()
File "/usr/local/lib/python3.9/site-packages/datapackage/package.py", line 534, in __build
updated_resource = Resource(descriptor,
File "/usr/local/lib/python3.9/site-packages/datapackage/resource.py", line 90, in __init__
self.__build()
File "/usr/local/lib/python3.9/site-packages/datapackage/resource.py", line 648, in __build
raise exception
File "/usr/local/lib/python3.9/site-packages/datapackage/resource.py", line 643, in __build
self.__profile.validate(self.__current_descriptor)
File "/usr/local/lib/python3.9/site-packages/datapackage/profile.py", line 89, in validate
raise exceptions.ValidationError(message, errors=errors)
datapackage.exceptions.ValidationError: There are 1 validation errors (see exception.errors)
Pipeline ID: knesset/ethics_committee_decisions
Pipeline ID: knesset/knesset_committee_decisions
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/knesset/knesset_legal_advisor.py", line 44, in flow
document = gcl.download(url, use_curl=True, outfile=filename)
File "/datapackage_pipelines_budgetkey/common/google_chrome.py", line 219, in download
assert False, 'Failed to download file, %r' % downloads
AssertionError: Failed to download file, ['']
DEBUG :[chan 13] Max packet in: 32768 bytes
DEBUG :[chan 13] Max packet out: 32768 bytes
DEBUG :Secsh channel 13 opened.
DEBUG :[chan 13] Sesch channel 13 request ok
DEBUG :[chan 13] EOF received (13)
DEBUG :[chan 13] EOF sent (13)
DEBUG :Dropping user packet because connection is dead.
Pipeline ID: knesset/knesset_legal_advisor
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/knesset/knesset_legal_advisor_letters.py", line 44, in flow
document = gcl.download(url, use_curl=True, outfile=filename)
File "/datapackage_pipelines_budgetkey/common/google_chrome.py", line 219, in download
assert False, 'Failed to download file, %r' % downloads
AssertionError: Failed to download file, ['']
DEBUG :[chan 13] Max packet in: 32768 bytes
DEBUG :[chan 13] Max packet out: 32768 bytes
DEBUG :Secsh channel 13 opened.
DEBUG :[chan 13] Sesch channel 13 request ok
DEBUG :[chan 13] EOF received (13)
DEBUG :[chan 13] EOF sent (13)
DEBUG :Dropping user packet because connection is dead.
Pipeline ID: knesset/knesset_legal_advisor_letters
Pipeline ID: lamas/municipal-data
Pipeline ID: lamas/municipality-to-city
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-bonds-and-options-holdings-changes
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-company-director-appointments
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-company-officer-list
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-company-stakeholder-list
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-company-vip-appointments
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates From ./maya/maya-notification-parse-updates :['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']
Pipeline ID: maya/maya-complete-notification-list
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-holdings-changes
Dependency unsuccessful :Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed
Pipeline ID: maya/maya-notification-parse-updates
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-reported-academic-degrees
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-reported-work-record
Dependency unsuccessful :Cannot run until dependency "./maya/scrape-maya-tase-companies" is successfully executed
Pipeline ID: maya/maya-tase-companies-current-management
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 79, in _process
self.datapackage = self.process_datapackage(self.datapackage)
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 112, in process_datapackage
raise self.exc
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 60, in describe
sample = list(itertools.islice(self.iterable, self.SAMPLE_SIZE))
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 89, in handle_iterable
for x in self.iterable:
File "/datapackage_pipelines_budgetkey/pipelines/maya/scrape_maya_notification_list.py", line 153, in scrape_maya_notification_list
first_date, last_date = get_existing_records()
File "/datapackage_pipelines_budgetkey/pipelines/maya/scrape_maya_notification_list.py", line 31, in get_existing_records
rs = con.execute(text("SELECT min(date), max(date) FROM maya_notifications where parser_version=:v"), v=PARSER_VERSION)
TypeError: execute() got an unexpected keyword argument 'v'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module>
spew_flow(flow, ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/utilities/flow_utils.py", line 46, in spew_flow
datastream = flow.datastream()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/flow.py", line 19, in datastream
return self._chain(ds)._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 86, in _process
self.raise_exception(exception)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 96, in raise_exception
raise error from cause
dataflows.base.exceptions.ProcessorError: Errored in processor iterable_loader in position #9: execute() got an unexpected keyword argument 'v'
Pipeline ID: maya/scrape-maya-notification-list
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/connection.py", line 72, in create_connection
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
File "/usr/local/lib/python3.9/socket.py", line 966, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 716, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 404, in _make_request
self._validate_conn(conn)
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 1061, in _validate_conn
conn.connect()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 363, in connect
self.sock = conn = self._new_conn()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 186, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x7d155f1ea7f0>: Failed to establish a new connection: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 644, in send
resp = conn.urlopen(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 802, in urlopen
retries = retries.increment(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/retry.py", line 594, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='info.tase.co.il', port=443): Max retries exceeded with url: /heb/marketdata/tase_companies/Pages/tase_companies.aspx (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7d155f1ea7f0>: Failed to establish a new connection: [Errno -2] Name or service not known'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 79, in _process
self.datapackage = self.process_datapackage(self.datapackage)
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 112, in process_datapackage
raise self.exc
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 60, in describe
sample = list(itertools.islice(self.iterable, self.SAMPLE_SIZE))
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 89, in handle_iterable
for x in self.iterable:
File "/datapackage_pipelines_budgetkey/pipelines/maya/scrape_maya_tase_companies.py", line 21, in scrape_maya_tase_companies
req = requests.get('https://info.tase.co.il/heb/marketdata/tase_companies/Pages/tase_companies.aspx')
File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 677, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPSConnectionPool(host='info.tase.co.il', port=443): Max retries exceeded with url: /heb/marketdata/tase_companies/Pages/tase_companies.aspx (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7d155f1ea7f0>: Failed to establish a new connection: [Errno -2] Name or service not known'))
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module>
spew_flow(flow, ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/utilities/flow_utils.py", line 46, in spew_flow
datastream = flow.datastream()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/flow.py", line 19, in datastream
return self._chain(ds)._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 86, in _process
self.raise_exception(exception)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 96, in raise_exception
raise error from cause
dataflows.base.exceptions.ProcessorError: Errored in processor iterable_loader in position #3: HTTPSConnectionPool(host='info.tase.co.il', port=443): Max retries exceeded with url: /heb/marketdata/tase_companies/Pages/tase_companies.aspx (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7d155f1ea7f0>: Failed to establish a new connection: [Errno -2] Name or service not known'))
Pipeline ID: maya/scrape-maya-tase-companies
Invalid dependency :Cannot run until dependency passes validation: ./people/all From ./people/all :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/all-appointments'] From ./people/all :['From ./people/company_appointments/all-appointments', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/media/themarker/themarker" is successfully executed']] From ./people/all :['From ./people/company_appointments/all-appointments', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-nominations']] From ./people/all :['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-nominations', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list']]] From ./people/all :['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-nominations', ['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']]]] From ./people/all :['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-nominations', ['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]]] From ./people/all :['From ./people/company_appointments/all-appointments', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-resignations']] From ./people/all :['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-resignations', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list']]] From ./people/all :['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-resignations', ['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']]]] From ./people/all :['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-resignations', ['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]]] From ./people/all :['Dependency unsuccessful', 'Cannot run until dependency "./people/political_candidates/election_candidates21" is successfully executed']
Pipeline ID: people/aggregated
Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/all-appointments From ./people/company_appointments/all-appointments :['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/media/themarker/themarker" is successfully executed'] From ./people/company_appointments/all-appointments :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-nominations'] From ./people/company_appointments/all-appointments :['From ./people/company_appointments/maya/maya-nominations', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list']] From ./people/company_appointments/all-appointments :['From ./people/company_appointments/maya/maya-nominations', ['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']]] From ./people/company_appointments/all-appointments :['From ./people/company_appointments/maya/maya-nominations', ['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]] From ./people/company_appointments/all-appointments :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-resignations'] From ./people/company_appointments/all-appointments :['From ./people/company_appointments/maya/maya-resignations', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list']] From ./people/company_appointments/all-appointments :['From ./people/company_appointments/maya/maya-resignations', ['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']]] From ./people/company_appointments/all-appointments :['From ./people/company_appointments/maya/maya-resignations', ['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]] Dependency unsuccessful :Cannot run until dependency "./people/political_candidates/election_candidates21" is successfully executed
Pipeline ID: people/all
Pipeline ID: people/association_founders/association-founders
Dependency unsuccessful :Cannot run until dependency "./people/company_appointments/media/themarker/themarker" is successfully executed Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-nominations From ./people/company_appointments/maya/maya-nominations :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list'] From ./people/company_appointments/maya/maya-nominations :['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']] From ./people/company_appointments/maya/maya-nominations :['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]] Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-resignations From ./people/company_appointments/maya/maya-resignations :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list'] From ./people/company_appointments/maya/maya-resignations :['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']] From ./people/company_appointments/maya/maya-resignations :['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]
Pipeline ID: people/company_appointments/all-appointments
Dependency unsuccessful :Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed
Pipeline ID: people/company_appointments/maya/collect-maya-forms-debug-stats
Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-nominations From ./people/company_appointments/maya/maya-nominations :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list'] From ./people/company_appointments/maya/maya-nominations :['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']] From ./people/company_appointments/maya/maya-nominations :['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]] Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-resignations From ./people/company_appointments/maya/maya-resignations :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list'] From ./people/company_appointments/maya/maya-resignations :['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']] From ./people/company_appointments/maya/maya-resignations :['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]
Pipeline ID: people/company_appointments/maya/correlate
Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates From ./people/company_appointments/maya/maya-notification-updates :['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']
Pipeline ID: people/company_appointments/maya/maya-list
Name, job and company description.
Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list From ./people/company_appointments/maya/maya-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates'] From ./people/company_appointments/maya/maya-list :['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]
Pipeline ID: people/company_appointments/maya/maya-nominations
scrape-notification-list
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 976, in json
return complexjson.loads(self.text, **kwargs)
File "/usr/local/lib/python3.9/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/local/lib/python3.9/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/local/lib/python3.9/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/pipelines/people/company_appointments/maya/scrape-notification-list.py", line 55, in _maya_api_call
return res.json()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 980, in json
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/pipelines/people/company_appointments/maya/scrape-notification-list.py", line 130, in <module>
spew(datapackage, [collect()])
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew
for rec in res:
File "/datapackage_pipelines_budgetkey/pipelines/people/company_appointments/maya/scrape-notification-list.py", line 113, in collect
yield from _collect_date_range(year_start, year_end)
File "/datapackage_pipelines_budgetkey/pipelines/people/company_appointments/maya/scrape-notification-list.py", line 75, in _collect_date_range
res = _maya_api_call(date_from, date_to, current_page)
File "/datapackage_pipelines_budgetkey/pipelines/people/company_appointments/maya/scrape-notification-list.py", line 57, in _maya_api_call
raise Exception("Failed to Call Maya API for date_from:{} date_to:{} page_num:{}".format(date_from, date_to, page_num)) from e
Exception: Failed to Call Maya API for date_from:2006-01-01 date_to:2007-01-01 page_num:0
Pipeline ID: people/company_appointments/maya/maya-notification-list
id, date, company, notification_type, s3_object_name, url
Dependency unsuccessful :Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed
Pipeline ID: people/company_appointments/maya/maya-notification-updates
Name, job and company description.
Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list From ./people/company_appointments/maya/maya-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates'] From ./people/company_appointments/maya/maya-list :['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]
Pipeline ID: people/company_appointments/maya/maya-resignations
רשימת כל המינויים והחברות מאתר כלכליסט
Pipeline ID: people/company_appointments/media/calcalist/calcalist
Name, job and company description.
stream_remote_resources
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 63, in load
bytes = _RemoteStream(source, self.__http_session, self.__http_timeout).open()
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 121, in open
self.seek(0)
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 143, in seek
self.__response.raise_for_status()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 1026, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 404 Client Error: Not Found for url: https://www.themarker.com/st/inter/DB/tm/2015/minuyim30.xlsx
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/stream_remote_resources.py", line 244, in <module>
rows = stream_reader(resource, url, ignore_missing or url == "", limit_rows,
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/stream_remote_resources.py", line 199, in stream_reader
schema, headers, columns, stream, close = get_opener(url, _resource)()
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/stream_remote_resources.py", line 175, in opener
_stream.open()
File "/usr/local/lib/python3.9/site-packages/tabulator/stream.py", line 425, in open
self.__parser.open(source, encoding=self.__encoding)
File "/usr/local/lib/python3.9/site-packages/tabulator/parsers/xlsx.py", line 81, in open
source_bytes = self.__loader.load(source, mode="b", encoding=encoding)
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 72, in load
raise exceptions.HTTPError(str(exception))
tabulator.exceptions.HTTPError: 404 Client Error: Not Found for url: https://www.themarker.com/st/inter/DB/tm/2015/minuyim30.xlsx
Pipeline ID: people/company_appointments/media/themarker/themarker
Pipeline ID: people/company_appointments/state_owned/correlate
Pipeline ID: people/company_appointments/state_owned/parse_pdf_resources
columns: date, url
Pipeline ID: people/company_appointments/state_owned/reports
Pipeline ID: people/company_appointments/state_owned/state-owned-nominations
Pipeline ID: people/municipal_appointments/municipal_representitives
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 79, in _process
self.datapackage = self.process_datapackage(self.datapackage)
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/set_type.py", line 71, in process_datapackage
assert added, 'Failed to find field {} in schema'.format(self.name)
AssertionError: Failed to find field re.compile('^party_letter$') in schema
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module>
spew_flow(flow, ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/utilities/flow_utils.py", line 46, in spew_flow
datastream = flow.datastream()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/flow.py", line 19, in datastream
return self._chain(ds)._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
[Previous line repeated 4 more times]
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 86, in _process
self.raise_exception(exception)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 96, in raise_exception
raise error from cause
dataflows.base.exceptions.ProcessorError: Errored in processor set_type in position #3: Failed to find field re.compile('^party_letter$') in schema
Pipeline ID: people/political_candidates/election_candidates21
Pipeline ID: people/political_donations/political_donations
Pipeline ID: people/procurement/procurement-individuals
Dependency unsuccessful :Cannot run until dependency "./procurement/calls_for_bids/gov-publications" is successfully executed Dependency unsuccessful :Cannot run until dependency "./procurement/calls_for_bids/gov-publications2" is successfully executed Dependency unsuccessful :Cannot run until dependency "./procurement/calls_for_bids/gov-publications-ministry-of-immigration" is successfully executed
Pipeline ID: procurement/calls_for_bids/calls-for-bids-all
Pipeline ID: procurement/calls_for_bids/calls-for-bids-moh
flow ERROR :Traceback (most recent call last): ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module> ERROR :spew_flow(flow, ctx) ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__ ERROR :spew(self.datapackage, self.resource_iterator, stats=self.stats) ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew ERROR :for rec in res: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/validate.py", line 55, in process_resource ERROR :yield from self.validator(res) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/validate.py", line 50, in func ERROR :yield from schema_validator(res.res, res, on_error=self.on_error) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/rows_processor.py", line 11, in process_resource ERROR :yield from self.func(resource) ERROR :File "/datapackage_pipelines_budgetkey/processors/gov_publications.py", line 53, in func ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :[Previous line repeated 1 more time] ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/filter_rows.py", line 19, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :[Previous line repeated 2 more times] ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 41, in process_resource ERROR :yield self.process_row(row) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/row_processor.py", line 11, in process_row ERROR :ret = self.func(row) ERROR :File "/datapackage_pipelines_budgetkey/processors/gov_publications.py", line 70, in f ERROR :row[field] = row[field][0] ERROR :IndexError ERROR :: ERROR :list index out of range
Pipeline ID: procurement/calls_for_bids/gov-publications
flow ERROR :Traceback (most recent call last): ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module> ERROR :spew_flow(flow, ctx) ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__ ERROR :spew(self.datapackage, self.resource_iterator, stats=self.stats) ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew ERROR :for rec in res: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/validate.py", line 55, in process_resource ERROR :yield from self.validator(res) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/validate.py", line 50, in func ERROR :yield from schema_validator(res.res, res, on_error=self.on_error) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/rows_processor.py", line 11, in process_resource ERROR :yield from self.func(resource) ERROR :File "/datapackage_pipelines_budgetkey/processors/gov_publications.py", line 53, in func ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :[Previous line repeated 1 more time] ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/filter_rows.py", line 19, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :[Previous line repeated 2 more times] ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 41, in process_resource ERROR :yield self.process_row(row) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/row_processor.py", line 11, in process_row ERROR :ret = self.func(row) ERROR :File "/datapackage_pipelines_budgetkey/processors/gov_publications.py", line 70, in f ERROR :row[field] = row[field][0] ERROR :IndexError ERROR :: ERROR :list index out of range
Pipeline ID: procurement/calls_for_bids/gov-publications-ministry-of-immigration
flow ERROR :Traceback (most recent call last): ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module> ERROR :spew_flow(flow, ctx) ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__ ERROR :spew(self.datapackage, self.resource_iterator, stats=self.stats) ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew ERROR :for rec in res: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/validate.py", line 55, in process_resource ERROR :yield from self.validator(res) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/validate.py", line 50, in func ERROR :yield from schema_validator(res.res, res, on_error=self.on_error) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/rows_processor.py", line 11, in process_resource ERROR :yield from self.func(resource) ERROR :File "/datapackage_pipelines_budgetkey/processors/gov_publications.py", line 53, in func ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :[Previous line repeated 1 more time] ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/filter_rows.py", line 19, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :[Previous line repeated 2 more times] ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 41, in process_resource ERROR :yield self.process_row(row) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/row_processor.py", line 11, in process_row ERROR :ret = self.func(row) ERROR :File "/datapackage_pipelines_budgetkey/processors/gov_publications.py", line 70, in f ERROR :row[field] = row[field][0] ERROR :IndexError ERROR :: ERROR :list index out of range
Pipeline ID: procurement/calls_for_bids/gov-publications2
Dependency unsuccessful :Cannot run until dependency "./procurement/calls_for_bids/jobiz-scraper" is successfully executed
Pipeline ID: procurement/calls_for_bids/jobiz-all
Invalid dependency :Cannot run until dependency passes validation: ./procurement/calls_for_bids/jobiz-all From ./procurement/calls_for_bids/jobiz-all :['Dependency unsuccessful', 'Cannot run until dependency "./procurement/calls_for_bids/jobiz-scraper" is successfully executed']
Pipeline ID: procurement/calls_for_bids/jobiz-calls-for-bids
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/connection.py", line 72, in create_connection
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
File "/usr/local/lib/python3.9/socket.py", line 966, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -5] No address associated with hostname
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 716, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 404, in _make_request
self._validate_conn(conn)
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 1061, in _validate_conn
conn.connect()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 363, in connect
self.sock = conn = self._new_conn()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 186, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x7fd503da5f10>: Failed to establish a new connection: [Errno -5] No address associated with hostname
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 644, in send
resp = conn.urlopen(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 802, in urlopen
retries = retries.increment(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/retry.py", line 594, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='jobiz.gov.il', port=443): Max retries exceeded with url: /ajax/results/%D7%94%D7%95%D7%93%D7%A2%D7%94%20%D7%A6%D7%99%D7%91%D7%95%D7%A8%D7%99%D7%AA/0?ie=0&typeie=%D7%94%D7%95%D7%93%D7%A2%D7%94+%D7%A6%D7%99%D7%91%D7%95%D7%A8%D7%99%D7%AA&search=Array (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fd503da5f10>: Failed to establish a new connection: [Errno -5] No address associated with hostname'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 79, in _process
self.datapackage = self.process_datapackage(self.datapackage)
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 112, in process_datapackage
raise self.exc
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 60, in describe
sample = list(itertools.islice(self.iterable, self.SAMPLE_SIZE))
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 89, in handle_iterable
for x in self.iterable:
File "/datapackage_pipelines_budgetkey/pipelines/procurement/calls_for_bids/jobiz.py", line 19, in fetch_results
content = requests.get(URL.format(index)).json()
File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 677, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPSConnectionPool(host='jobiz.gov.il', port=443): Max retries exceeded with url: /ajax/results/%D7%94%D7%95%D7%93%D7%A2%D7%94%20%D7%A6%D7%99%D7%91%D7%95%D7%A8%D7%99%D7%AA/0?ie=0&typeie=%D7%94%D7%95%D7%93%D7%A2%D7%94+%D7%A6%D7%99%D7%91%D7%95%D7%A8%D7%99%D7%AA&search=Array (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fd503da5f10>: Failed to establish a new connection: [Errno -5] No address associated with hostname'))
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module>
spew_flow(flow, ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/utilities/flow_utils.py", line 46, in spew_flow
datastream = flow.datastream()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/flow.py", line 19, in datastream
return self._chain(ds)._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
[Previous line repeated 5 more times]
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 86, in _process
self.raise_exception(exception)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 96, in raise_exception
raise error from cause
dataflows.base.exceptions.ProcessorError: Errored in processor iterable_loader in position #1: HTTPSConnectionPool(host='jobiz.gov.il', port=443): Max retries exceeded with url: /ajax/results/%D7%94%D7%95%D7%93%D7%A2%D7%94%20%D7%A6%D7%99%D7%91%D7%95%D7%A8%D7%99%D7%AA/0?ie=0&typeie=%D7%94%D7%95%D7%93%D7%A2%D7%94+%D7%A6%D7%99%D7%91%D7%95%D7%A8%D7%99%D7%AA&search=Array (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fd503da5f10>: Failed to establish a new connection: [Errno -5] No address associated with hostname'))
Pipeline ID: procurement/calls_for_bids/jobiz-scraper
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/procurement/municipal/datacity-tender-fetcher.py", line 55, in flow
DF.dump_to_sql(dict(
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/to_sql.py", line 67, in __init__
with self.engine.connect():
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3295, in connect
return self._connection_cls(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 146, in __init__
Connection._handle_dbapi_exception_noconnection(
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2450, in _handle_dbapi_exception_noconnection
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
(Background on this error at: https://sqlalche.me/e/20/e3q8)
Pipeline ID: procurement/municipal/fetcher
אנו סוכמים מידע מתוך הדוחות הרבעוניים לתוך טבלה המכילה את המידע העדכני ביותר.
Pipeline ID: procurement/spending/latest-contract-spending
Pipeline ID: procurement/spending/quarterly-contract-spending-report-uris
כל משרדי הממשלה ויחידות הסמך (שמחוברות למערכת מרכבה) צריכים לפרסם אחת לרבעון (תוך 45 יום מסוף הרבעון) דו״ח התקשרויות לפי פורמט מוגדר היטב הממומש כדו״ח מובנה במערכת המרכבה. כל המשרדים מדווחים עצמאית והדו״חות עולים לאתר היחידה לחופש המידע, משם אנו אוספים אותם. התוצר של השרשר הזה הוא פירוט כל הדיווחים של כל המשרדים.
Pipeline ID: procurement/spending/quarterly-contract-spending-reports
Pipeline ID: procurement/spending/quarterly-contract-spending-reports-data
Pipeline ID: procurement/spending/spending-by-entity
dump.to_sql
ERROR :Traceback (most recent call last):
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2127, in _exec_insertmany_context
ERROR :dialect.do_execute(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 952, in do_execute
ERROR :cursor.execute(statement, parameters)
ERROR :psycopg2
ERROR :.
ERROR :OperationalError
ERROR ::
ERROR :server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
ERROR :The above exception was the direct cause of the following exception:
ERROR :Traceback (most recent call last):
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/dump/to_sql.py", line 15, in <module>
ERROR :spew_flow(flow(ctx.parameters), ctx)
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__
ERROR :spew(self.datapackage, self.resource_iterator, stats=self.stats)
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew
ERROR :for rec in res:
ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/dumper_base.py", line 69, in row_counter
ERROR :for row in iterator:
ERROR :File "/usr/local/lib/python3.9/site-packages/tableschema_sql/writer.py", line 55, in write
ERROR :for wr in self.__insert(connection):
ERROR :File "/usr/local/lib/python3.9/site-packages/tableschema_sql/writer.py", line 86, in __insert
ERROR :connection.execute(statement, self.__buffer)
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1421, in execute
ERROR :return meth(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 526, in _execute_on_connection
ERROR :return connection._execute_clauseelement(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1643, in _execute_clauseelement
ERROR :ret = self._execute_context(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1846, in _execute_context
ERROR :return self._exec_insertmany_context(dialect, context)
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2135, in _exec_insertmany_context
ERROR :self._handle_dbapi_exception(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2365, in _handle_dbapi_exception
ERROR :raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2127, in _exec_insertmany_context
ERROR :dialect.do_execute(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 952, in do_execute
ERROR :cursor.execute(statement, parameters)
ERROR :sqlalchemy.exc
ERROR :.
ERROR :OperationalError
ERROR ::
ERROR :(psycopg2.OperationalError) server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
[SQL: INSERT INTO procurement_tenders_all (publication_id, publisher, publisher_unit, tender_type_he, tender_id, description, status, publication_date, last_update_date, claim_date, supplier, supplier_id, start_date, end_date, subjects, regulation, decisio ... 585058 characters truncated ... _title__999)s, %(reason__999)s, %(documents__999)s::JSONB, %(contact__999)s, %(contact_email__999)s)]
[parameters: {'supplier__0': None, 'start_date__0': None, 'publisher_unit__0': 'משרד הכלכלה - הרשות להגבלים עסקיים', 'supplier_id__0': '512615964', 'last_update_date__0': datetime.date(2013, 6, 18), 'subjects__0': 'מחשוב ותקשורת', 'publication_id__0': '543014', 'publisher__0': 'משרד הכלכלה והתעשייה', 'status__0': 'פורסם', 'volume__0': None, 'tender_type_he__0': 'פרסום כוונה להתקשרות', 'publication_date__0': datetime.date(2013, 6, 18), 'reason__0': 'תקנה 3(29) - ספק יחיד', 'documents__0': '[]', 'tender_id__0': 'none', 'source_currency__0': 'ILS', 'decision__0': 'טרם התקבלה החלטת ועדה', 'page_title__0': 'עובדי מחלקת חקירות משתמשים במכשירי PTT של חברת הוט מובייל (לשעבר מירס) אשר משמשים לצרכים מבצעים. מכשיר זה ה', 'contact_email__0': None, 'approving_level__0': 'הפטור', 'page_url__0': 'https://mr.gov.il/ilgstorefront/he/p/543014', 'description__0': 'עובדי מחלקת חקירות משתמשים במכשירי PTT של חברת הוט מובייל (לשעבר מירס) אשר משמשים לצרכים מבצעים. מכשיר זה ה', 'regulation__0': 'תקנה 3(29) - ספק יחיד', 'tender_type__0': 'exemptions', 'claim_date__0': datetime.datetime(2013, 7, 7, 0, 0), 'contact__0': None, 'end_date__0': None, 'supplier__1': None, 'start_date__1': None, 'publisher_unit__1': 'משרד החקלאות ובטחון המזון', 'supplier_id__1': '512017989', 'last_update_date__1': datetime.date(2013, 6, 18), 'subjects__1': 'שירותים אישיים ומקומיים', 'publication_id__1': '543015', 'publisher__1': 'משרד החקלאות ובטחון המזון', 'status__1': 'פורסם', 'volume__1': None, 'tender_type_he__1': 'פרסום כוונה להתקשרות', 'publication_date__1': datetime.date(2013, 6, 18), 'reason__1': 'תקנה 3(29) - ספק יחיד', 'documents__1': '[]', 'tender_id__1': 'none', 'source_currency__1': 'ILS', 'decision__1': 'טרם התקבלה החלטת ועדה', 'page_title__1': 'רכישת מוצרי מזון (כיבוד) עבור לשכת שר החקלאות', 'contact_email__1': None, 'approving_level__1': 'הפטור', 'page_url__1': 'https://mr.gov.il/ilgstorefront/he/p/543015', 'description__1': 'רכישת מוצרי מזון (כיבוד) עבור לשכת שר החקלאות', 'regulation__1': 'תקנה 3(29) - ספק יחיד' ... 26900 parameters truncated ... 'last_update_date__998': datetime.date(2013, 7, 28), 'subjects__998': 'שירותי כביסה וגיהוץ', 'publication_id__998': '544096', 'publisher__998': 'משרד הבריאות', 'status__998': 'פורסם', 'volume__998': Decimal('192240.00'), 'tender_type_he__998': 'התקשרות בפטור במכרז או בהליך תחרותי אחר', 'publication_date__998': datetime.date(2013, 7, 28), 'reason__998': 'תקנה 3(2) - התקשרות דחופה למניעת נזק של ממש', 'documents__998': '[]', 'tender_id__998': 'none', 'source_currency__998': 'ILS', 'decision__998': 'התקשרות מאושרת', 'page_title__998': 'חברת יונידרס - שרותי כביסה בבית החולים הלל יפה', 'contact_email__998': None, 'approving_level__998': 'מכרזים משרדית', 'page_url__998': 'https://mr.gov.il/ilgstorefront/he/p/544096', 'description__998': 'חברת יונידרס - שרותי כביסה בבית החולים הלל יפה', 'regulation__998': 'תקנה 3(2) - התקשרות דחופה למניעת נזק של ממש', 'tender_type__998': 'exemptions', 'claim_date__998': None, 'contact__998': None, 'end_date__998': datetime.date(2014, 1, 22), 'supplier__999': None, 'start_date__999': None, 'publisher_unit__999': 'משרד הרווחה', 'supplier_id__999': '512762584', 'last_update_date__999': datetime.date(2013, 7, 28), 'subjects__999': 'מחשוב ותקשורת', 'publication_id__999': '544097', 'publisher__999': 'משרד הרווחה', 'status__999': 'פורסם', 'volume__999': None, 'tender_type_he__999': 'פרסום כוונה להתקשרות', 'publication_date__999': datetime.date(2013, 7, 28), 'reason__999': 'תקנה 3(29) - ספק יחיד', 'documents__999': '[]', 'tender_id__999': 'none', 'source_currency__999': 'ILS', 'decision__999': 'טרם התקבלה החלטת ועדה', 'page_title__999': 'משרד הרווחה והשירותים החברתיים מעוניין להתקשר עם חברת CA בע"מ בדבר שדרוג תוכנות מתוצרתה בסביבת ה- MF תחת מערכ', 'contact_email__999': None, 'approving_level__999': 'הפטור', 'page_url__999': 'https://mr.gov.il/ilgstorefront/he/p/544097', 'description__999': 'משרד הרווחה והשירותים החברתיים מעוניין להתקשר עם חברת CA בע"מ בדבר שדרוג תוכנות מתוצרתה בסביבת ה- MF תחת מערכ', 'regulation__999': 'תקנה 3(29) - ספק יחיד', 'tender_type__999': 'exemptions', 'claim_date__999': datetime.datetime(2013, 8, 14, 0, 0), 'contact__999': None, 'end_date__999': None}]
(Background on this error at: https://sqlalche.me/e/20/e3q8)
Pipeline ID: procurement/tenders/all
Pipeline ID: procurement/tenders/processed
Pipeline ID: reports/all
Pipeline ID: reports/ngos/ngo-activity-report
Pipeline ID: reports/ngos/ngo-district-report
Pipeline ID: simpledb/process_simple_dbs
Pipeline ID: support_programs/all-support-programs
Dependency unsuccessful :Cannot run until dependency "./supports/supports-2026" is successfully executed
Pipeline ID: supports/all
Pipeline ID: supports/by-payment-year
Pipeline ID: supports/by-request-year
sample
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/sample.py", line 11, in <module>
res_name = parameters.get('resource', datapackage['resources'][0]['name'])
IndexError: list index out of range
Pipeline ID: supports/criteria/support-criteria
Pipeline ID: supports/criteria/support-criteria-doj
Pipeline ID: supports/supports-2004
Pipeline ID: supports/supports-2005
Pipeline ID: supports/supports-2006
Pipeline ID: supports/supports-2007
Pipeline ID: supports/supports-2008
Pipeline ID: supports/supports-2009
Pipeline ID: supports/supports-2010
Pipeline ID: supports/supports-2011
Pipeline ID: supports/supports-2012
Pipeline ID: supports/supports-2013
Pipeline ID: supports/supports-2014
Pipeline ID: supports/supports-2015
Pipeline ID: supports/supports-2016
Pipeline ID: supports/supports-2017
Pipeline ID: supports/supports-2018
Pipeline ID: supports/supports-2019
Pipeline ID: supports/supports-2020
Pipeline ID: supports/supports-2021
Pipeline ID: supports/supports-2022
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 181, in flow
DF.load(wrapper(year), format='csv',
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 21, in wrapper
return scraper(gcd, year)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 164, in scraper
return gcd.download('https://next.obudget.org/datapackages/', outfile=gcd.list_downloads()[0])
File "/datapackage_pipelines_budgetkey/common/google_chrome.py", line 204, in download
assert response.status_code == 200
AssertionError
WARNING :FAILED to teardown google-chrome
Pipeline ID: supports/supports-2023
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 181, in flow
DF.load(wrapper(year), format='csv',
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 21, in wrapper
return scraper(gcd, year)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 164, in scraper
return gcd.download('https://next.obudget.org/datapackages/', outfile=gcd.list_downloads()[0])
File "/datapackage_pipelines_budgetkey/common/google_chrome.py", line 204, in download
assert response.status_code == 200
AssertionError
WARNING :FAILED to teardown google-chrome
Pipeline ID: supports/supports-2024
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 181, in flow
DF.load(wrapper(year), format='csv',
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 21, in wrapper
return scraper(gcd, year)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 164, in scraper
return gcd.download('https://next.obudget.org/datapackages/', outfile=gcd.list_downloads()[0])
File "/datapackage_pipelines_budgetkey/common/google_chrome.py", line 204, in download
assert response.status_code == 200
AssertionError
WARNING :FAILED to teardown google-chrome
Pipeline ID: supports/supports-2025
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 181, in flow
DF.load(wrapper(year), format='csv',
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 21, in wrapper
return scraper(gcd, year)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 164, in scraper
return gcd.download('https://next.obudget.org/datapackages/', outfile=gcd.list_downloads()[0])
File "/datapackage_pipelines_budgetkey/common/google_chrome.py", line 204, in download
assert response.status_code == 200
AssertionError
WARNING :FAILED to teardown google-chrome
Pipeline ID: supports/supports-2026
Pipeline ID: supports/supports-by-entity
Pipeline ID: supports/with-entities
Pipeline ID: supports/with-keys
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 63, in load
bytes = _RemoteStream(source, self.__http_session, self.__http_timeout).open()
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 121, in open
self.seek(0)
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 143, in seek
self.__response.raise_for_status()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 1026, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 502 Server Error: Bad Gateway for url: https://data-input.obudget.org/api/datarecords/hierarchy
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/load.py", line 125, in process_datapackage
return self.safe_process_datapackage(dp)
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/load.py", line 186, in safe_process_datapackage
stream: Stream = Stream(self.load_source, **self.options).open()
File "/usr/local/lib/python3.9/site-packages/tabulator/stream.py", line 425, in open
self.__parser.open(source, encoding=self.__encoding)
File "/usr/local/lib/python3.9/site-packages/tabulator/parsers/json.py", line 40, in open
self.__bytes = self.__loader.load(source, mode='b', encoding=encoding)
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 72, in load
raise exceptions.HTTPError(str(exception))
tabulator.exceptions.HTTPError: 502 Server Error: Bad Gateway for url: https://data-input.obudget.org/api/datarecords/hierarchy
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 79, in _process
self.datapackage = self.process_datapackage(self.datapackage)
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/load.py", line 127, in process_datapackage
raise SourceLoadError('Failed to load source {!r} and options {!r}: {}'
dataflows.base.exceptions.SourceLoadError: Failed to load source 'https://data-input.obudget.org/api/datarecords/hierarchy' and options {'format': 'json', 'property': 'result', 'custom_parsers': {'xml': <class 'dataflows.processors.parsers.xml_parser.XMLParser'>, 'excel-xml': <class 'dataflows.processors.parsers.excel_xml_parser.ExcelXMLParser'>, 'sql': <class 'dataflows.processors.parsers.sql_parser.ExtendedSQLParser'>, 'geojson': <class 'dataflows.processors.parsers.geojson_parser.GeoJsonParser'>}, 'ignore_blank_headers': True, 'skip_rows': [{'type': 'preset', 'value': 'auto'}], 'headers': 1, 'sample_size': 1000}: 502 Server Error: Bad Gateway for url: https://data-input.obudget.org/api/datarecords/hierarchy
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module>
spew_flow(flow, ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/utilities/flow_utils.py", line 46, in spew_flow
datastream = flow.datastream()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/flow.py", line 19, in datastream
return self._chain(ds)._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
[Previous line repeated 16 more times]
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 86, in _process
self.raise_exception(exception)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 97, in raise_exception
raise cause
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 79, in _process
self.datapackage = self.process_datapackage(self.datapackage)
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 112, in process_datapackage
raise self.exc
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 60, in describe
sample = list(itertools.islice(self.iterable, self.SAMPLE_SIZE))
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 89, in handle_iterable
for x in self.iterable:
File "/datapackage_pipelines_budgetkey/pipelines/units/social_services/social_services.py", line 21, in all_units
hierarchies = datarecords('hierarchy')
File "/datapackage_pipelines_budgetkey/pipelines/units/social_services/social_services.py", line 7, in datarecords
DF.Flow(
File "/usr/local/lib/python3.9/site-packages/dataflows/base/flow.py", line 13, in results
return self._chain().results(on_error=on_error)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 127, in results
ds, results = self.safe_process(return_results=True, on_error=on_error)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 119, in safe_process
self.raise_exception(exception)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 97, in raise_exception
raise cause
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 102, in safe_process
ds = self._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 86, in _process
self.raise_exception(exception)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 96, in raise_exception
raise error from cause
dataflows.base.exceptions.ProcessorError: Errored in processor load in position #1: Failed to load source 'https://data-input.obudget.org/api/datarecords/hierarchy' and options {'format': 'json', 'property': 'result', 'custom_parsers': {'xml': <class 'dataflows.processors.parsers.xml_parser.XMLParser'>, 'excel-xml': <class 'dataflows.processors.parsers.excel_xml_parser.ExcelXMLParser'>, 'sql': <class 'dataflows.processors.parsers.sql_parser.ExtendedSQLParser'>, 'geojson': <class 'dataflows.processors.parsers.geojson_parser.GeoJsonParser'>}, 'ignore_blank_headers': True, 'skip_rows': [{'type': 'preset', 'value': 'auto'}], 'headers': 1, 'sample_size': 1000}: 502 Server Error: Bad Gateway for url: https://data-input.obudget.org/api/datarecords/hierarchy
Pipeline ID: units/social_services/social_services
Invalid dependency :Cannot run until dependency passes validation: ./procurement/calls_for_bids/calls-for-bids-all From ./procurement/calls_for_bids/calls-for-bids-all :['Dependency unsuccessful', 'Cannot run until dependency "./procurement/calls_for_bids/gov-publications" is successfully executed'] From ./procurement/calls_for_bids/calls-for-bids-all :['Dependency unsuccessful', 'Cannot run until dependency "./procurement/calls_for_bids/gov-publications2" is successfully executed'] From ./procurement/calls_for_bids/calls-for-bids-all :['Dependency unsuccessful', 'Cannot run until dependency "./procurement/calls_for_bids/gov-publications-ministry-of-immigration" is successfully executed']
Pipeline ID: budgetkey/elasticsearch/index_calls_for_bids
Invalid dependency :Cannot run until dependency passes validation: ./people/aggregated From ./people/aggregated :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/all'] From ./people/aggregated :['From ./people/all', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/all-appointments']] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/media/themarker/themarker" is successfully executed']]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-nominations']]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-nominations', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list']]]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-nominations', ['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']]]]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-nominations', ['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]]]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-resignations']]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-resignations', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list']]]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-resignations', ['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']]]]] From ./people/aggregated :['From ./people/all', ['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-resignations', ['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]]]] From ./people/aggregated :['From ./people/all', ['Dependency unsuccessful', 'Cannot run until dependency "./people/political_candidates/election_candidates21" is successfully executed']]
Pipeline ID: budgetkey/elasticsearch/index_people
Dependency unsuccessful :Cannot run until dependency "./supports/criteria/support-criteria" is successfully executed
Pipeline ID: budgetkey/elasticsearch/index_support_criteria
Dependency unsuccessful :Cannot run until dependency "./budgetkey/elasticsearch/index_national_budget_changes" is successfully executed Dependency unsuccessful :Cannot run until dependency "./budgetkey/elasticsearch/index_entities" is successfully executed
Pipeline ID: budgetkey/elasticsearch/sitemaps
Dependency unsuccessful :Cannot run until dependency "./entities/special/registry" is successfully executed
Pipeline ID: entities/all
Dependency unsuccessful :Cannot run until dependency "./entities/safety-orders/safety-orders" is successfully executed
Pipeline ID: entities/safety-orders/safety-orders-by-entity
Dependency unsuccessful :Cannot run until dependency "./ezvonot/ezvonot_accounts" is successfully executed Dependency unsuccessful :Cannot run until dependency "./ezvonot/ezvonot_cases" is successfully executed Dependency unsuccessful :Cannot run until dependency "./ezvonot/ezvonot_purposes" is successfully executed Dependency unsuccessful :Cannot run until dependency "./ezvonot/ezvonot_decisions" is successfully executed
Pipeline ID: ezvonot/ezvonot_decisions_join
Dependency unsuccessful :Cannot run until dependency "./facilities/labor/facilities_labor" is successfully executed
Pipeline ID: facilities/all/facilities_all
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-bonds-and-options-holdings-changes
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-company-director-appointments
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-company-officer-list
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-company-stakeholder-list
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-company-vip-appointments
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates From ./maya/maya-notification-parse-updates :['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']
Pipeline ID: maya/maya-complete-notification-list
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-holdings-changes
Dependency unsuccessful :Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed
Pipeline ID: maya/maya-notification-parse-updates
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-reported-academic-degrees
Invalid dependency :Cannot run until dependency passes validation: ./maya/maya-complete-notification-list From ./maya/maya-complete-notification-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./maya/maya-notification-parse-updates'] From ./maya/maya-complete-notification-list :['From ./maya/maya-notification-parse-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./maya/scrape-maya-notification-list" is successfully executed']]
Pipeline ID: maya/maya-reported-work-record
Dependency unsuccessful :Cannot run until dependency "./maya/scrape-maya-tase-companies" is successfully executed
Pipeline ID: maya/maya-tase-companies-current-management
Invalid dependency :Cannot run until dependency passes validation: ./people/all From ./people/all :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/all-appointments'] From ./people/all :['From ./people/company_appointments/all-appointments', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/media/themarker/themarker" is successfully executed']] From ./people/all :['From ./people/company_appointments/all-appointments', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-nominations']] From ./people/all :['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-nominations', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list']]] From ./people/all :['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-nominations', ['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']]]] From ./people/all :['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-nominations', ['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]]] From ./people/all :['From ./people/company_appointments/all-appointments', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-resignations']] From ./people/all :['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-resignations', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list']]] From ./people/all :['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-resignations', ['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']]]] From ./people/all :['From ./people/company_appointments/all-appointments', ['From ./people/company_appointments/maya/maya-resignations', ['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]]] From ./people/all :['Dependency unsuccessful', 'Cannot run until dependency "./people/political_candidates/election_candidates21" is successfully executed']
Pipeline ID: people/aggregated
Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/all-appointments From ./people/company_appointments/all-appointments :['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/media/themarker/themarker" is successfully executed'] From ./people/company_appointments/all-appointments :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-nominations'] From ./people/company_appointments/all-appointments :['From ./people/company_appointments/maya/maya-nominations', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list']] From ./people/company_appointments/all-appointments :['From ./people/company_appointments/maya/maya-nominations', ['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']]] From ./people/company_appointments/all-appointments :['From ./people/company_appointments/maya/maya-nominations', ['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]] From ./people/company_appointments/all-appointments :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-resignations'] From ./people/company_appointments/all-appointments :['From ./people/company_appointments/maya/maya-resignations', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list']] From ./people/company_appointments/all-appointments :['From ./people/company_appointments/maya/maya-resignations', ['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']]] From ./people/company_appointments/all-appointments :['From ./people/company_appointments/maya/maya-resignations', ['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]] Dependency unsuccessful :Cannot run until dependency "./people/political_candidates/election_candidates21" is successfully executed
Pipeline ID: people/all
Dependency unsuccessful :Cannot run until dependency "./people/company_appointments/media/themarker/themarker" is successfully executed Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-nominations From ./people/company_appointments/maya/maya-nominations :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list'] From ./people/company_appointments/maya/maya-nominations :['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']] From ./people/company_appointments/maya/maya-nominations :['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]] Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-resignations From ./people/company_appointments/maya/maya-resignations :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list'] From ./people/company_appointments/maya/maya-resignations :['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']] From ./people/company_appointments/maya/maya-resignations :['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]
Pipeline ID: people/company_appointments/all-appointments
Dependency unsuccessful :Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed
Pipeline ID: people/company_appointments/maya/collect-maya-forms-debug-stats
Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-nominations From ./people/company_appointments/maya/maya-nominations :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list'] From ./people/company_appointments/maya/maya-nominations :['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']] From ./people/company_appointments/maya/maya-nominations :['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]] Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-resignations From ./people/company_appointments/maya/maya-resignations :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list'] From ./people/company_appointments/maya/maya-resignations :['From ./people/company_appointments/maya/maya-list', ['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates']] From ./people/company_appointments/maya/maya-resignations :['From ./people/company_appointments/maya/maya-list', ['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]]
Pipeline ID: people/company_appointments/maya/correlate
Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates From ./people/company_appointments/maya/maya-notification-updates :['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']
Pipeline ID: people/company_appointments/maya/maya-list
Name, job and company description.
Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list From ./people/company_appointments/maya/maya-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates'] From ./people/company_appointments/maya/maya-list :['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]
Pipeline ID: people/company_appointments/maya/maya-nominations
id, date, company, notification_type, s3_object_name, url
Dependency unsuccessful :Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed
Pipeline ID: people/company_appointments/maya/maya-notification-updates
Name, job and company description.
Invalid dependency :Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-list From ./people/company_appointments/maya/maya-list :['Invalid dependency', 'Cannot run until dependency passes validation: ./people/company_appointments/maya/maya-notification-updates'] From ./people/company_appointments/maya/maya-list :['From ./people/company_appointments/maya/maya-notification-updates', ['Dependency unsuccessful', 'Cannot run until dependency "./people/company_appointments/maya/maya-notification-list" is successfully executed']]
Pipeline ID: people/company_appointments/maya/maya-resignations
Dependency unsuccessful :Cannot run until dependency "./procurement/calls_for_bids/gov-publications" is successfully executed Dependency unsuccessful :Cannot run until dependency "./procurement/calls_for_bids/gov-publications2" is successfully executed Dependency unsuccessful :Cannot run until dependency "./procurement/calls_for_bids/gov-publications-ministry-of-immigration" is successfully executed
Pipeline ID: procurement/calls_for_bids/calls-for-bids-all
Dependency unsuccessful :Cannot run until dependency "./procurement/calls_for_bids/jobiz-scraper" is successfully executed
Pipeline ID: procurement/calls_for_bids/jobiz-all
Invalid dependency :Cannot run until dependency passes validation: ./procurement/calls_for_bids/jobiz-all From ./procurement/calls_for_bids/jobiz-all :['Dependency unsuccessful', 'Cannot run until dependency "./procurement/calls_for_bids/jobiz-scraper" is successfully executed']
Pipeline ID: procurement/calls_for_bids/jobiz-calls-for-bids
Dependency unsuccessful :Cannot run until dependency "./supports/supports-2026" is successfully executed
Pipeline ID: supports/all
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/activities/social_services/measurements/collect.py", line 311, in flow
DF.dump_to_sql(dict(
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/to_sql.py", line 67, in __init__
with self.engine.connect():
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3295, in connect
return self._connection_cls(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 146, in __init__
Connection._handle_dbapi_exception_noconnection(
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2450, in _handle_dbapi_exception_noconnection
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
(Background on this error at: https://sqlalche.me/e/20/e3q8)
Pipeline ID: activities/social_services/measurements/collect
scraper
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/connection.py", line 95, in create_connection
raise err
File "/usr/local/lib/python3.9/site-packages/urllib3/util/connection.py", line 85, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 716, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 416, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 244, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/usr/local/lib/python3.9/http/client.py", line 1285, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/lib/python3.9/http/client.py", line 1331, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.9/http/client.py", line 1280, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.9/http/client.py", line 1040, in _send_output
self.send(msg)
File "/usr/local/lib/python3.9/http/client.py", line 980, in send
self.connect()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 205, in connect
conn = self._new_conn()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 186, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7fb30de60580>: Failed to establish a new connection: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 644, in send
resp = conn.urlopen(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 802, in urlopen
retries = retries.increment(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/retry.py", line 594, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='www.boi.org.il', port=80): Max retries exceeded with url: /he/DataAndStatistics/_layouts/boi/handlers/WebPartHandler.aspx?wp=SeriesSearchBySubject&lang=he-IL&PaggingSize=100&Level=4&sId=1 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fb30de60580>: Failed to establish a new connection: [Errno 111] Connection refused'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/pipelines/bank_of_israel/scraper.py", line 124, in <module>
series_sum = get_all_series()
File "/datapackage_pipelines_budgetkey/pipelines/bank_of_israel/scraper.py", line 23, in get_all_series
series = get_series(sid)
File "/datapackage_pipelines_budgetkey/pipelines/bank_of_israel/scraper.py", line 34, in get_series
response = session.get(SEARCH_PAGE_RESULTS_URL.format(sid=sid))
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 677, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='www.boi.org.il', port=80): Max retries exceeded with url: /he/DataAndStatistics/_layouts/boi/handlers/WebPartHandler.aspx?wp=SeriesSearchBySubject&lang=he-IL&PaggingSize=100&Level=4&sId=1 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fb30de60580>: Failed to establish a new connection: [Errno 111] Connection refused'))
Pipeline ID: bank_of_israel/bank_of_israel
flow
ERROR :/usr/local/lib/python3.9/site-packages/dataflows/processors/join.py:189: UserWarning: For the `join` processor the `full=True` flag is deprecated. Please use the "mode" parameter instead.
warnings.warn(
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/budget/municipal/datacity-budget-fetcher.py", line 177, in flow
DF.dump_to_sql(dict(
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/to_sql.py", line 67, in __init__
with self.engine.connect():
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3295, in connect
return self._connection_cls(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 146, in __init__
Connection._handle_dbapi_exception_noconnection(
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2450, in _handle_dbapi_exception_noconnection
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
(Background on this error at: https://sqlalche.me/e/20/e3q8)
Pipeline ID: budget/municipal/fetcher
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/budget/national/changes/original/committee-zipfile.py", line 14, in flow
archive_ = gcl.download(WEIRD_ZIP_FILE, outfile=f'p{YEAR}.zip')
File "/datapackage_pipelines_budgetkey/common/google_chrome.py", line 219, in download
assert False, 'Failed to download file, %r' % downloads
AssertionError: Failed to download file, ['']
DEBUG :[chan 14] Max packet in: 32768 bytes
DEBUG :[chan 14] Max packet out: 32768 bytes
DEBUG :Secsh channel 14 opened.
DEBUG :[chan 14] Sesch channel 14 request ok
DEBUG :[chan 14] EOF received (14)
DEBUG :[chan 14] EOF sent (14)
DEBUG :Dropping user packet because connection is dead.
Pipeline ID: budget/national/changes/original/committee-zipfile
dump_to_es
ERROR :DUMP TO ES ERROR BadRequestError(400, 'illegal_argument_exception', 'mapper [details.pob] cannot be changed from type [text] to [long]')
ERROR :TB
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/dump_to_es.py", line 163, in <module>
spew_flow(flow(ctx.parameters), ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__
spew(self.datapackage, self.resource_iterator, stats=self.stats)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 68, in spew
for res in resources_iterator:
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 68, in <genexpr>
res_iter = (it if isinstance(it, ResourceWrapper) else ResourceWrapper(res, it)
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/dumper_base.py", line 82, in process_resources
ret = self.process_resource(
File "/usr/local/lib/python3.9/site-packages/dataflows_elasticsearch/processors/dump_to_elasticsearch.py", line 69, in process_resource
storage.create(index_name, res.descriptor['schema'],
File "/usr/local/lib/python3.9/site-packages/tableschema_elasticsearch/storage.py", line 123, in create
self.put_mapping(bucket, descriptor, index_name, mapping_generator_cls)
File "/usr/local/lib/python3.9/site-packages/tableschema_elasticsearch/storage.py", line 85, in put_mapping
self.__es.indices.put_mapping(index=index_name, **mapping)
File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/utils.py", line 458, in wrapped
return api(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/indices.py", line 4137, in put_mapping
return self.perform_request( # type: ignore[return-value]
File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/_base.py", line 422, in perform_request
return self._client.perform_request(
File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/_base.py", line 271, in perform_request
response = self._perform_request(
File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/_base.py", line 351, in _perform_request
raise HTTP_EXCEPTIONS.get(meta.status, ApiError)(
elasticsearch.BadRequestError: BadRequestError(400, 'illegal_argument_exception', 'mapper [details.pob] cannot be changed from type [text] to [long]')
ERROR :Traceback (most recent call last):
ERROR :File "/datapackage_pipelines_budgetkey/processors/dump_to_es.py", line 163, in <module>
ERROR :spew_flow(flow(ctx.parameters), ctx)
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__
ERROR :spew(self.datapackage, self.resource_iterator, stats=self.stats)
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 68, in spew
ERROR :for res in resources_iterator:
ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 68, in <genexpr>
ERROR :res_iter = (it if isinstance(it, ResourceWrapper) else ResourceWrapper(res, it)
ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/dumper_base.py", line 82, in process_resources
ERROR :ret = self.process_resource(
ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows_elasticsearch/processors/dump_to_elasticsearch.py", line 69, in process_resource
ERROR :storage.create(index_name, res.descriptor['schema'],
ERROR :File "/usr/local/lib/python3.9/site-packages/tableschema_elasticsearch/storage.py", line 123, in create
ERROR :self.put_mapping(bucket, descriptor, index_name, mapping_generator_cls)
ERROR :File "/usr/local/lib/python3.9/site-packages/tableschema_elasticsearch/storage.py", line 85, in put_mapping
ERROR :self.__es.indices.put_mapping(index=index_name, **mapping)
ERROR :File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/utils.py", line 458, in wrapped
ERROR :return api(*args, **kwargs)
ERROR :File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/indices.py", line 4137, in put_mapping
ERROR :return self.perform_request( # type: ignore[return-value]
ERROR :File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/_base.py", line 422, in perform_request
ERROR :return self._client.perform_request(
ERROR :File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/_base.py", line 271, in perform_request
ERROR :response = self._perform_request(
ERROR :File "/usr/local/lib/python3.9/site-packages/elasticsearch/_sync/client/_base.py", line 351, in _perform_request
ERROR :raise HTTP_EXCEPTIONS.get(meta.status, ApiError)(
ERROR :elasticsearch
ERROR :.
ERROR :BadRequestError
ERROR ::
ERROR :BadRequestError(400, 'illegal_argument_exception', 'mapper [details.pob] cannot be changed from type [text] to [long]')
Pipeline ID: budgetkey/elasticsearch/index_entities
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/openai/_base_client.py", line 1113, in request
response.raise_for_status()
File "/usr/local/lib/python3.9/site-packages/httpx/_models.py", line 829, in raise_for_status
raise HTTPStatusError(message, request=request, response=self)
httpx.HTTPStatusError: Client error '400 Bad Request' for url 'https://api.openai.com/v1/embeddings'
For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/400
DEBUG :Not retrying
DEBUG :Re-raising status error
ERROR :Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module>
spew_flow(flow, ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__
spew(self.datapackage, self.resource_iterator, stats=self.stats)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew
for rec in res:
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/rows_processor.py", line 11, in process_resource
yield from self.func(resource)
File "/datapackage_pipelines_budgetkey/pipelines/budgetkey/elasticsearch/chunker.py", line 47, in func
hit, embedding = embed(chunk)
File "/datapackage_pipelines_budgetkey/common/cached_openai.py", line 42, in embed
embedding = client.embeddings.create(
File "/usr/local/lib/python3.9/site-packages/openai/resources/embeddings.py", line 136, in create
return self._post(
File "/usr/local/lib/python3.9/site-packages/openai/_base_client.py", line 1360, in post
return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))
File "/usr/local/lib/python3.9/site-packages/openai/_base_client.py", line 1133, in request
raise self._make_status_error_from_response(err.response) from None
openai.BadRequestError: Error code: 400 - {'error': {'message': "Invalid 'input': maximum context length is 8192 tokens.", 'type': 'invalid_request_error', 'param': None, 'code': None}}
DEBUG :close.started
DEBUG :close.complete
Pipeline ID: budgetkey/elasticsearch/index_national_budget_changes
get_transactions
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/pipelines/donations/get_transactions.py", line 150, in <module>
spew(dp, process_resources(res_iter))
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew
for rec in res:
File "/datapackage_pipelines_budgetkey/pipelines/donations/get_transactions.py", line 109, in get_transactions
for resp in self.get_for_candidate(cid):
File "/datapackage_pipelines_budgetkey/pipelines/donations/get_transactions.py", line 95, in get_for_candidate
resp = self.get_for_range(cid, range_start, range_end)
File "/datapackage_pipelines_budgetkey/pipelines/donations/get_transactions.py", line 70, in get_for_range
assert len(resp) == 6
TypeError: object of type 'NoneType' has no len()
Pipeline ID: donations/transactions
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/education/tochniyot/scraper.py", line 103, in flow
return Flow(scrape(),
File "/datapackage_pipelines_budgetkey/pipelines/education/tochniyot/scraper.py", line 65, in scrape
education_programs_count = get_education_programs_count()
File "/datapackage_pipelines_budgetkey/pipelines/education/tochniyot/scraper.py", line 56, in get_education_programs_count
res = send_tochniyot_request()
File "/datapackage_pipelines_budgetkey/pipelines/education/tochniyot/scraper.py", line 80, in send_tochniyot_request
resp.raise_for_status()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 1026, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 500 Server Error: Internal Server Error for url: https://apps.education.gov.il/TyhNet/ClientWs/TochnitCh.asmx/IturTochnitChByMeafyenim
Pipeline ID: education/tochniyot/tochniyot
רשימת כלל החברות הפעילות, מתעדכן שבועית.
dump.to_sql
ERROR :Traceback (most recent call last):
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2127, in _exec_insertmany_context
ERROR :dialect.do_execute(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 952, in do_execute
ERROR :cursor.execute(statement, parameters)
ERROR :psycopg2
ERROR :.
ERROR :OperationalError
ERROR ::
ERROR :server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
ERROR :The above exception was the direct cause of the following exception:
ERROR :Traceback (most recent call last):
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/dump/to_sql.py", line 15, in <module>
ERROR :spew_flow(flow(ctx.parameters), ctx)
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__
ERROR :spew(self.datapackage, self.resource_iterator, stats=self.stats)
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew
ERROR :for rec in res:
ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/dumper_base.py", line 69, in row_counter
ERROR :for row in iterator:
ERROR :File "/usr/local/lib/python3.9/site-packages/tableschema_sql/writer.py", line 55, in write
ERROR :for wr in self.__insert(connection):
ERROR :File "/usr/local/lib/python3.9/site-packages/tableschema_sql/writer.py", line 86, in __insert
ERROR :connection.execute(statement, self.__buffer)
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1421, in execute
ERROR :return meth(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 526, in _execute_on_connection
ERROR :return connection._execute_clauseelement(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1643, in _execute_clauseelement
ERROR :ret = self._execute_context(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1846, in _execute_context
ERROR :return self._exec_insertmany_context(dialect, context)
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2135, in _exec_insertmany_context
ERROR :self._handle_dbapi_exception(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2365, in _handle_dbapi_exception
ERROR :raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2127, in _exec_insertmany_context
ERROR :dialect.do_execute(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 952, in do_execute
ERROR :cursor.execute(statement, parameters)
ERROR :sqlalchemy.exc
ERROR :.
ERROR :OperationalError
ERROR ::
ERROR :(psycopg2.OperationalError) server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
[SQL: INSERT INTO company_details (id, company_name, company_name_eng, company_type, company_status, company_description, company_goal, company_registration_date, company_is_government, company_limit, company_is_mafera, company_last_report_year, company_ci ... 534839 characters truncated ... pany_pob__999)s, %(company_country__999)s, %(company_located_at__999)s, %(company_sub_status__999)s)]
[parameters: {'company_located_at__0': None, 'company_pob__0': None, 'company_name__0': 'לונה נייל בע״מ', 'id__0': '512957408', 'company_postal_code__0': '6433812', 'company_description__0': None, 'company_country__0': 'ישראל', 'company_limit__0': 'מוגבלת', 'company_sub_status__0': None, 'company_registration_date__0': datetime.date(2000, 5, 23), 'company_name_eng__0': None, 'company_type__0': 'ישראלית חברה פרטית', 'company_last_report_year__0': None, 'company_street__0': "המלך ג'ורג'", 'company_status__0': 'פעילה', 'company_goal__0': 'לעסוק בכל עיסוק חוקי', 'company_is_government__0': False, 'company_is_mafera__0': True, 'company_street_number__0': '88', 'company_city__0': 'תל אביב - יפו', 'company_located_at__1': 'מגדל לוינשטיין קומה 29', 'company_pob__1': None, 'company_name__1': 'טק-איז (ישראל) בע״מ', 'id__1': '512957416', 'company_postal_code__1': '66182', 'company_description__1': None, 'company_country__1': 'ישראל', 'company_limit__1': 'מוגבלת', 'company_sub_status__1': None, 'company_registration_date__1': datetime.date(2000, 5, 23), 'company_name_eng__1': 'TECC-IS (ISRAEL) LTD', 'company_type__1': 'ישראלית חברה פרטית', 'company_last_report_year__1': 2003, 'company_street__1': 'מנחם בגין', 'company_status__1': 'מחוסלת מרצון', 'company_goal__1': 'לעסוק בכל עיסוק חוקי', 'company_is_government__1': False, 'company_is_mafera__1': None, 'company_street_number__1': '23', 'company_city__1': 'תל אביב - יפו', 'company_located_at__2': 'א.ת צפוני לוד', 'company_pob__2': None, 'company_name__2': 'קשת יהודה עבודות בניין ואחזקות בע״מ', 'id__2': '512957424', 'company_postal_code__2': '7120517', 'company_description__2': None, 'company_country__2': 'ישראל', 'company_limit__2': 'מוגבלת', 'company_sub_status__2': None, 'company_registration_date__2': datetime.date(2000, 5, 23) ... 19900 parameters truncated ... 'company_name_eng__997': None, 'company_type__997': 'ישראלית חברה פרטית', 'company_last_report_year__997': None, 'company_street__997': 'אייזק כהן', 'company_status__997': 'מחוסלת מרצון', 'company_goal__997': 'לעסוק בכל עיסוק חוקי', 'company_is_government__997': False, 'company_is_mafera__997': None, 'company_street_number__997': '275', 'company_city__997': 'צפת', 'company_located_at__998': None, 'company_pob__998': None, 'company_name__998': 'שער הטכניון חברה לניהול בע״מ', 'id__998': '512967712', 'company_postal_code__998': '2751021', 'company_description__998': None, 'company_country__998': 'ישראל', 'company_limit__998': 'מוגבלת', 'company_sub_status__998': None, 'company_registration_date__998': datetime.date(2000, 6, 15), 'company_name_eng__998': None, 'company_type__998': 'ישראלית חברה פרטית', 'company_last_report_year__998': 2025, 'company_street__998': 'חרושת', 'company_status__998': 'פעילה', 'company_goal__998': 'לעסוק בכל עיסוק חוקי', 'company_is_government__998': False, 'company_is_mafera__998': None, 'company_street_number__998': '4', 'company_city__998': 'קריית ביאליק', 'company_located_at__999': None, 'company_pob__999': None, 'company_name__999': 'אורות מטבחים בע״מ', 'id__999': '512967720', 'company_postal_code__999': '5960602', 'company_description__999': None, 'company_country__999': 'ישראל', 'company_limit__999': 'מוגבלת', 'company_sub_status__999': None, 'company_registration_date__999': datetime.date(2000, 6, 15), 'company_name_eng__999': None, 'company_type__999': 'ישראלית חברה פרטית', 'company_last_report_year__999': 2004, 'company_street__999': 'הפלדה', 'company_status__999': 'פעילה', 'company_goal__999': 'לעסוק בכל עיסוק חוקי', 'company_is_government__999': False, 'company_is_mafera__999': True, 'company_street_number__999': '4', 'company_city__999': 'בת ים'}]
(Background on this error at: https://sqlalche.me/e/20/e3q8)
Pipeline ID: entities/companies/registrar/scraper
fingerprint
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/fingerprint.py", line 201, in <module>
resource = next(iter(filter(
StopIteration
Pipeline ID: entities/foi_orgs/foi_orgs
scraper
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/pipelines/entities/moital/scraper.py", line 79, in <module>
spew(datapackage, [scrape()])
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew
for rec in res:
File "/datapackage_pipelines_budgetkey/pipelines/entities/moital/scraper.py", line 32, in scrape
desired_capabilities=DesiredCapabilities.PHANTOMJS)
AttributeError: type object 'DesiredCapabilities' has no attribute 'PHANTOMJS'
Pipeline ID: entities/moital/moital_service_providers
stream_remote_resources
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/connection.py", line 72, in create_connection
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
File "/usr/local/lib/python3.9/socket.py", line 966, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 716, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 416, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 244, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/usr/local/lib/python3.9/http/client.py", line 1285, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/lib/python3.9/http/client.py", line 1331, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.9/http/client.py", line 1280, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.9/http/client.py", line 1040, in _send_output
self.send(msg)
File "/usr/local/lib/python3.9/http/client.py", line 980, in send
self.connect()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 205, in connect
conn = self._new_conn()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 186, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7f7bf46da7f0>: Failed to establish a new connection: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 644, in send
resp = conn.urlopen(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 802, in urlopen
retries = retries.increment(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/retry.py", line 594, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='employment.molsa.gov.il', port=80): Max retries exceeded with url: /Employment/SafetyAndHealth/Enforcement/Documents/%D7%A6%D7%95%D7%95%D7%99%20%D7%91%D7%98%D7%99%D7%97%D7%95%D7%AA.xlsx (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f7bf46da7f0>: Failed to establish a new connection: [Errno -2] Name or service not known'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 63, in load
bytes = _RemoteStream(source, self.__http_session, self.__http_timeout).open()
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 121, in open
self.seek(0)
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 142, in seek
self.__response = self.__session.get(self.__source, stream=True, timeout=self.__timeout)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 677, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='employment.molsa.gov.il', port=80): Max retries exceeded with url: /Employment/SafetyAndHealth/Enforcement/Documents/%D7%A6%D7%95%D7%95%D7%99%20%D7%91%D7%98%D7%99%D7%97%D7%95%D7%AA.xlsx (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f7bf46da7f0>: Failed to establish a new connection: [Errno -2] Name or service not known'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/stream_remote_resources.py", line 244, in <module>
rows = stream_reader(resource, url, ignore_missing or url == "", limit_rows,
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/stream_remote_resources.py", line 199, in stream_reader
schema, headers, columns, stream, close = get_opener(url, _resource)()
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/stream_remote_resources.py", line 175, in opener
_stream.open()
File "/usr/local/lib/python3.9/site-packages/tabulator/stream.py", line 425, in open
self.__parser.open(source, encoding=self.__encoding)
File "/usr/local/lib/python3.9/site-packages/tabulator/parsers/xlsx.py", line 81, in open
source_bytes = self.__loader.load(source, mode="b", encoding=encoding)
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 72, in load
raise exceptions.HTTPError(str(exception))
tabulator.exceptions.HTTPError: HTTPConnectionPool(host='employment.molsa.gov.il', port=80): Max retries exceeded with url: /Employment/SafetyAndHealth/Enforcement/Documents/%D7%A6%D7%95%D7%95%D7%99%20%D7%91%D7%98%D7%99%D7%97%D7%95%D7%AA.xlsx (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f7bf46da7f0>: Failed to establish a new connection: [Errno -2] Name or service not known'))
Pipeline ID: entities/safety-orders/safety-orders
dump.to_sql
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/dump/to_sql.py", line 15, in <module>
spew_flow(flow(ctx.parameters), ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/lib/dump_to_sql.py", line 8, in flow
dump_to_sql(
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/to_sql.py", line 67, in __init__
with self.engine.connect():
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3295, in connect
return self._connection_cls(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 146, in __init__
Connection._handle_dbapi_exception_noconnection(
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2450, in _handle_dbapi_exception_noconnection
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
(Background on this error at: https://sqlalche.me/e/20/e3q8)
Pipeline ID: entities/special/registry
data_gov_il_resource
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 976, in json
return complexjson.loads(self.text, **kwargs)
File "/usr/local/lib/python3.9/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/local/lib/python3.9/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/local/lib/python3.9/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 23, in search_dataset
results = requests.get(PACKAGE_GET_URL + dataset_name, headers=HEADERS).json()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 980, in json
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 92, in get_resource
dataset = search_dataset(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 27, in search_dataset
results = gcd.json(PACKAGE_GET_URL + dataset_name)
AttributeError: 'function' object has no attribute 'json'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 40, in get_page
assert test(page)
UnboundLocalError: local variable 'page' referenced before assignment
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 66, in <module>
spew_flow(flow(ctx.parameters), ctx)
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 47, in flow
url, path = get_resource(gcd, dataset_name, resource_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 94, in get_resource
dataset = get_dataset_html(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 49, in get_dataset_html
page = get_page(gcd, PACKAGE_PAGE_URL + dataset_name, lambda page: 'resource-item' in page)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 42, in get_page
gcd.driver.get(url)
AttributeError: 'function' object has no attribute 'driver'
Pipeline ID: ezvonot/ezvonot_accounts
data_gov_il_resource
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 976, in json
return complexjson.loads(self.text, **kwargs)
File "/usr/local/lib/python3.9/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/local/lib/python3.9/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/local/lib/python3.9/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 23, in search_dataset
results = requests.get(PACKAGE_GET_URL + dataset_name, headers=HEADERS).json()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 980, in json
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 92, in get_resource
dataset = search_dataset(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 27, in search_dataset
results = gcd.json(PACKAGE_GET_URL + dataset_name)
AttributeError: 'function' object has no attribute 'json'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 40, in get_page
assert test(page)
UnboundLocalError: local variable 'page' referenced before assignment
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 66, in <module>
spew_flow(flow(ctx.parameters), ctx)
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 47, in flow
url, path = get_resource(gcd, dataset_name, resource_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 94, in get_resource
dataset = get_dataset_html(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 49, in get_dataset_html
page = get_page(gcd, PACKAGE_PAGE_URL + dataset_name, lambda page: 'resource-item' in page)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 42, in get_page
gcd.driver.get(url)
AttributeError: 'function' object has no attribute 'driver'
Pipeline ID: ezvonot/ezvonot_cases
data_gov_il_resource
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 976, in json
return complexjson.loads(self.text, **kwargs)
File "/usr/local/lib/python3.9/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/local/lib/python3.9/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/local/lib/python3.9/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 23, in search_dataset
results = requests.get(PACKAGE_GET_URL + dataset_name, headers=HEADERS).json()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 980, in json
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 92, in get_resource
dataset = search_dataset(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 27, in search_dataset
results = gcd.json(PACKAGE_GET_URL + dataset_name)
AttributeError: 'function' object has no attribute 'json'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 40, in get_page
assert test(page)
UnboundLocalError: local variable 'page' referenced before assignment
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 66, in <module>
spew_flow(flow(ctx.parameters), ctx)
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 47, in flow
url, path = get_resource(gcd, dataset_name, resource_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 94, in get_resource
dataset = get_dataset_html(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 49, in get_dataset_html
page = get_page(gcd, PACKAGE_PAGE_URL + dataset_name, lambda page: 'resource-item' in page)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 42, in get_page
gcd.driver.get(url)
AttributeError: 'function' object has no attribute 'driver'
Pipeline ID: ezvonot/ezvonot_decisions
data_gov_il_resource
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 976, in json
return complexjson.loads(self.text, **kwargs)
File "/usr/local/lib/python3.9/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/local/lib/python3.9/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/local/lib/python3.9/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 23, in search_dataset
results = requests.get(PACKAGE_GET_URL + dataset_name, headers=HEADERS).json()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 980, in json
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 92, in get_resource
dataset = search_dataset(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 27, in search_dataset
results = gcd.json(PACKAGE_GET_URL + dataset_name)
AttributeError: 'function' object has no attribute 'json'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 40, in get_page
assert test(page)
UnboundLocalError: local variable 'page' referenced before assignment
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 66, in <module>
spew_flow(flow(ctx.parameters), ctx)
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 47, in flow
url, path = get_resource(gcd, dataset_name, resource_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 94, in get_resource
dataset = get_dataset_html(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 49, in get_dataset_html
page = get_page(gcd, PACKAGE_PAGE_URL + dataset_name, lambda page: 'resource-item' in page)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 42, in get_page
gcd.driver.get(url)
AttributeError: 'function' object has no attribute 'driver'
Pipeline ID: ezvonot/ezvonot_deposits
data_gov_il_resource
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 976, in json
return complexjson.loads(self.text, **kwargs)
File "/usr/local/lib/python3.9/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/local/lib/python3.9/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/local/lib/python3.9/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 23, in search_dataset
results = requests.get(PACKAGE_GET_URL + dataset_name, headers=HEADERS).json()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 980, in json
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 92, in get_resource
dataset = search_dataset(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 27, in search_dataset
results = gcd.json(PACKAGE_GET_URL + dataset_name)
AttributeError: 'function' object has no attribute 'json'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 40, in get_page
assert test(page)
UnboundLocalError: local variable 'page' referenced before assignment
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 66, in <module>
spew_flow(flow(ctx.parameters), ctx)
File "/datapackage_pipelines_budgetkey/processors/data_gov_il_resource.py", line 47, in flow
url, path = get_resource(gcd, dataset_name, resource_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 94, in get_resource
dataset = get_dataset_html(gcd, dataset_name)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 49, in get_dataset_html
page = get_page(gcd, PACKAGE_PAGE_URL + dataset_name, lambda page: 'resource-item' in page)
File "/datapackage_pipelines_budgetkey/common/data_gov_il.py", line 42, in get_page
gcd.driver.get(url)
AttributeError: 'function' object has no attribute 'driver'
Pipeline ID: ezvonot/ezvonot_purposes
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/facilities/labor/mol.py", line 77, in flow
DF.dump_to_sql(dict(
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/to_sql.py", line 67, in __init__
with self.engine.connect():
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3295, in connect
return self._connection_cls(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 146, in __init__
Connection._handle_dbapi_exception_noconnection(
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2450, in _handle_dbapi_exception_noconnection
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
(Background on this error at: https://sqlalche.me/e/20/e3q8)
Pipeline ID: facilities/labor/facilities_labor
(sink)
ERROR :Data Package validation error: Descriptor validation error: [] should be non-empty at "schema/fields" in descriptor and at "properties/schema/properties/fields/minItems" in profile
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/manager/../lib/internal/sink.py", line 24, in <module>
params, dp, res_iter = ingest()
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 186, in ingest
params, datapackage, resource_iterator = _ingest(debug=debug)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 44, in _ingest
datapackage, resource_iterator, dependency_dp = process_input(sys.stdin, validate, debug)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/input_processor.py", line 88, in process_input
datapackage.validate(dp_to_validate)
File "/usr/local/lib/python3.9/site-packages/datapackage/validate.py", line 28, in validate
Package(descriptor, strict=True)
File "/usr/local/lib/python3.9/site-packages/datapackage/package.py", line 126, in __init__
self.__build()
File "/usr/local/lib/python3.9/site-packages/datapackage/package.py", line 534, in __build
updated_resource = Resource(descriptor,
File "/usr/local/lib/python3.9/site-packages/datapackage/resource.py", line 90, in __init__
self.__build()
File "/usr/local/lib/python3.9/site-packages/datapackage/resource.py", line 648, in __build
raise exception
File "/usr/local/lib/python3.9/site-packages/datapackage/resource.py", line 643, in __build
self.__profile.validate(self.__current_descriptor)
File "/usr/local/lib/python3.9/site-packages/datapackage/profile.py", line 89, in validate
raise exceptions.ValidationError(message, errors=errors)
datapackage.exceptions.ValidationError: There are 1 validation errors (see exception.errors)
Pipeline ID: knesset/ethics_committee_decisions
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/knesset/knesset_legal_advisor.py", line 44, in flow
document = gcl.download(url, use_curl=True, outfile=filename)
File "/datapackage_pipelines_budgetkey/common/google_chrome.py", line 219, in download
assert False, 'Failed to download file, %r' % downloads
AssertionError: Failed to download file, ['']
DEBUG :[chan 13] Max packet in: 32768 bytes
DEBUG :[chan 13] Max packet out: 32768 bytes
DEBUG :Secsh channel 13 opened.
DEBUG :[chan 13] Sesch channel 13 request ok
DEBUG :[chan 13] EOF received (13)
DEBUG :[chan 13] EOF sent (13)
DEBUG :Dropping user packet because connection is dead.
Pipeline ID: knesset/knesset_legal_advisor
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/knesset/knesset_legal_advisor_letters.py", line 44, in flow
document = gcl.download(url, use_curl=True, outfile=filename)
File "/datapackage_pipelines_budgetkey/common/google_chrome.py", line 219, in download
assert False, 'Failed to download file, %r' % downloads
AssertionError: Failed to download file, ['']
DEBUG :[chan 13] Max packet in: 32768 bytes
DEBUG :[chan 13] Max packet out: 32768 bytes
DEBUG :Secsh channel 13 opened.
DEBUG :[chan 13] Sesch channel 13 request ok
DEBUG :[chan 13] EOF received (13)
DEBUG :[chan 13] EOF sent (13)
DEBUG :Dropping user packet because connection is dead.
Pipeline ID: knesset/knesset_legal_advisor_letters
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 79, in _process
self.datapackage = self.process_datapackage(self.datapackage)
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 112, in process_datapackage
raise self.exc
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 60, in describe
sample = list(itertools.islice(self.iterable, self.SAMPLE_SIZE))
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 89, in handle_iterable
for x in self.iterable:
File "/datapackage_pipelines_budgetkey/pipelines/maya/scrape_maya_notification_list.py", line 153, in scrape_maya_notification_list
first_date, last_date = get_existing_records()
File "/datapackage_pipelines_budgetkey/pipelines/maya/scrape_maya_notification_list.py", line 31, in get_existing_records
rs = con.execute(text("SELECT min(date), max(date) FROM maya_notifications where parser_version=:v"), v=PARSER_VERSION)
TypeError: execute() got an unexpected keyword argument 'v'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module>
spew_flow(flow, ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/utilities/flow_utils.py", line 46, in spew_flow
datastream = flow.datastream()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/flow.py", line 19, in datastream
return self._chain(ds)._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 86, in _process
self.raise_exception(exception)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 96, in raise_exception
raise error from cause
dataflows.base.exceptions.ProcessorError: Errored in processor iterable_loader in position #9: execute() got an unexpected keyword argument 'v'
Pipeline ID: maya/scrape-maya-notification-list
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/connection.py", line 72, in create_connection
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
File "/usr/local/lib/python3.9/socket.py", line 966, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 716, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 404, in _make_request
self._validate_conn(conn)
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 1061, in _validate_conn
conn.connect()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 363, in connect
self.sock = conn = self._new_conn()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 186, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x7d155f1ea7f0>: Failed to establish a new connection: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 644, in send
resp = conn.urlopen(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 802, in urlopen
retries = retries.increment(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/retry.py", line 594, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='info.tase.co.il', port=443): Max retries exceeded with url: /heb/marketdata/tase_companies/Pages/tase_companies.aspx (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7d155f1ea7f0>: Failed to establish a new connection: [Errno -2] Name or service not known'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 79, in _process
self.datapackage = self.process_datapackage(self.datapackage)
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 112, in process_datapackage
raise self.exc
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 60, in describe
sample = list(itertools.islice(self.iterable, self.SAMPLE_SIZE))
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 89, in handle_iterable
for x in self.iterable:
File "/datapackage_pipelines_budgetkey/pipelines/maya/scrape_maya_tase_companies.py", line 21, in scrape_maya_tase_companies
req = requests.get('https://info.tase.co.il/heb/marketdata/tase_companies/Pages/tase_companies.aspx')
File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 677, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPSConnectionPool(host='info.tase.co.il', port=443): Max retries exceeded with url: /heb/marketdata/tase_companies/Pages/tase_companies.aspx (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7d155f1ea7f0>: Failed to establish a new connection: [Errno -2] Name or service not known'))
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module>
spew_flow(flow, ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/utilities/flow_utils.py", line 46, in spew_flow
datastream = flow.datastream()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/flow.py", line 19, in datastream
return self._chain(ds)._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 86, in _process
self.raise_exception(exception)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 96, in raise_exception
raise error from cause
dataflows.base.exceptions.ProcessorError: Errored in processor iterable_loader in position #3: HTTPSConnectionPool(host='info.tase.co.il', port=443): Max retries exceeded with url: /heb/marketdata/tase_companies/Pages/tase_companies.aspx (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7d155f1ea7f0>: Failed to establish a new connection: [Errno -2] Name or service not known'))
Pipeline ID: maya/scrape-maya-tase-companies
scrape-notification-list
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 976, in json
return complexjson.loads(self.text, **kwargs)
File "/usr/local/lib/python3.9/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/local/lib/python3.9/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/local/lib/python3.9/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/pipelines/people/company_appointments/maya/scrape-notification-list.py", line 55, in _maya_api_call
return res.json()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 980, in json
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/pipelines/people/company_appointments/maya/scrape-notification-list.py", line 130, in <module>
spew(datapackage, [collect()])
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew
for rec in res:
File "/datapackage_pipelines_budgetkey/pipelines/people/company_appointments/maya/scrape-notification-list.py", line 113, in collect
yield from _collect_date_range(year_start, year_end)
File "/datapackage_pipelines_budgetkey/pipelines/people/company_appointments/maya/scrape-notification-list.py", line 75, in _collect_date_range
res = _maya_api_call(date_from, date_to, current_page)
File "/datapackage_pipelines_budgetkey/pipelines/people/company_appointments/maya/scrape-notification-list.py", line 57, in _maya_api_call
raise Exception("Failed to Call Maya API for date_from:{} date_to:{} page_num:{}".format(date_from, date_to, page_num)) from e
Exception: Failed to Call Maya API for date_from:2006-01-01 date_to:2007-01-01 page_num:0
Pipeline ID: people/company_appointments/maya/maya-notification-list
Name, job and company description.
stream_remote_resources
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 63, in load
bytes = _RemoteStream(source, self.__http_session, self.__http_timeout).open()
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 121, in open
self.seek(0)
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 143, in seek
self.__response.raise_for_status()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 1026, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 404 Client Error: Not Found for url: https://www.themarker.com/st/inter/DB/tm/2015/minuyim30.xlsx
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/stream_remote_resources.py", line 244, in <module>
rows = stream_reader(resource, url, ignore_missing or url == "", limit_rows,
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/stream_remote_resources.py", line 199, in stream_reader
schema, headers, columns, stream, close = get_opener(url, _resource)()
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/stream_remote_resources.py", line 175, in opener
_stream.open()
File "/usr/local/lib/python3.9/site-packages/tabulator/stream.py", line 425, in open
self.__parser.open(source, encoding=self.__encoding)
File "/usr/local/lib/python3.9/site-packages/tabulator/parsers/xlsx.py", line 81, in open
source_bytes = self.__loader.load(source, mode="b", encoding=encoding)
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 72, in load
raise exceptions.HTTPError(str(exception))
tabulator.exceptions.HTTPError: 404 Client Error: Not Found for url: https://www.themarker.com/st/inter/DB/tm/2015/minuyim30.xlsx
Pipeline ID: people/company_appointments/media/themarker/themarker
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 79, in _process
self.datapackage = self.process_datapackage(self.datapackage)
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/set_type.py", line 71, in process_datapackage
assert added, 'Failed to find field {} in schema'.format(self.name)
AssertionError: Failed to find field re.compile('^party_letter$') in schema
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module>
spew_flow(flow, ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/utilities/flow_utils.py", line 46, in spew_flow
datastream = flow.datastream()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/flow.py", line 19, in datastream
return self._chain(ds)._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
[Previous line repeated 4 more times]
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 86, in _process
self.raise_exception(exception)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 96, in raise_exception
raise error from cause
dataflows.base.exceptions.ProcessorError: Errored in processor set_type in position #3: Failed to find field re.compile('^party_letter$') in schema
Pipeline ID: people/political_candidates/election_candidates21
flow ERROR :Traceback (most recent call last): ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module> ERROR :spew_flow(flow, ctx) ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__ ERROR :spew(self.datapackage, self.resource_iterator, stats=self.stats) ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew ERROR :for rec in res: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/validate.py", line 55, in process_resource ERROR :yield from self.validator(res) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/validate.py", line 50, in func ERROR :yield from schema_validator(res.res, res, on_error=self.on_error) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/rows_processor.py", line 11, in process_resource ERROR :yield from self.func(resource) ERROR :File "/datapackage_pipelines_budgetkey/processors/gov_publications.py", line 53, in func ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :[Previous line repeated 1 more time] ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/filter_rows.py", line 19, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :[Previous line repeated 2 more times] ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 41, in process_resource ERROR :yield self.process_row(row) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/row_processor.py", line 11, in process_row ERROR :ret = self.func(row) ERROR :File "/datapackage_pipelines_budgetkey/processors/gov_publications.py", line 70, in f ERROR :row[field] = row[field][0] ERROR :IndexError ERROR :: ERROR :list index out of range
Pipeline ID: procurement/calls_for_bids/gov-publications
flow ERROR :Traceback (most recent call last): ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module> ERROR :spew_flow(flow, ctx) ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__ ERROR :spew(self.datapackage, self.resource_iterator, stats=self.stats) ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew ERROR :for rec in res: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/validate.py", line 55, in process_resource ERROR :yield from self.validator(res) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/validate.py", line 50, in func ERROR :yield from schema_validator(res.res, res, on_error=self.on_error) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/rows_processor.py", line 11, in process_resource ERROR :yield from self.func(resource) ERROR :File "/datapackage_pipelines_budgetkey/processors/gov_publications.py", line 53, in func ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :[Previous line repeated 1 more time] ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/filter_rows.py", line 19, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :[Previous line repeated 2 more times] ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 41, in process_resource ERROR :yield self.process_row(row) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/row_processor.py", line 11, in process_row ERROR :ret = self.func(row) ERROR :File "/datapackage_pipelines_budgetkey/processors/gov_publications.py", line 70, in f ERROR :row[field] = row[field][0] ERROR :IndexError ERROR :: ERROR :list index out of range
Pipeline ID: procurement/calls_for_bids/gov-publications-ministry-of-immigration
flow ERROR :Traceback (most recent call last): ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module> ERROR :spew_flow(flow, ctx) ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__ ERROR :spew(self.datapackage, self.resource_iterator, stats=self.stats) ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew ERROR :for rec in res: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/validate.py", line 55, in process_resource ERROR :yield from self.validator(res) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/validate.py", line 50, in func ERROR :yield from schema_validator(res.res, res, on_error=self.on_error) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/rows_processor.py", line 11, in process_resource ERROR :yield from self.func(resource) ERROR :File "/datapackage_pipelines_budgetkey/processors/gov_publications.py", line 53, in func ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :[Previous line repeated 1 more time] ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/filter_rows.py", line 19, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/add_computed_field.py", line 38, in process_resource ERROR :for row in rows: ERROR :[Previous line repeated 2 more times] ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 40, in process_resource ERROR :for row in resource: ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/schema_validator.py", line 68, in schema_validator ERROR :for i, row in enumerate(iterator): ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 41, in process_resource ERROR :yield self.process_row(row) ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/row_processor.py", line 11, in process_row ERROR :ret = self.func(row) ERROR :File "/datapackage_pipelines_budgetkey/processors/gov_publications.py", line 70, in f ERROR :row[field] = row[field][0] ERROR :IndexError ERROR :: ERROR :list index out of range
Pipeline ID: procurement/calls_for_bids/gov-publications2
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/connection.py", line 72, in create_connection
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
File "/usr/local/lib/python3.9/socket.py", line 966, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -5] No address associated with hostname
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 716, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 404, in _make_request
self._validate_conn(conn)
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 1061, in _validate_conn
conn.connect()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 363, in connect
self.sock = conn = self._new_conn()
File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 186, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x7fd503da5f10>: Failed to establish a new connection: [Errno -5] No address associated with hostname
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 644, in send
resp = conn.urlopen(
File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 802, in urlopen
retries = retries.increment(
File "/usr/local/lib/python3.9/site-packages/urllib3/util/retry.py", line 594, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='jobiz.gov.il', port=443): Max retries exceeded with url: /ajax/results/%D7%94%D7%95%D7%93%D7%A2%D7%94%20%D7%A6%D7%99%D7%91%D7%95%D7%A8%D7%99%D7%AA/0?ie=0&typeie=%D7%94%D7%95%D7%93%D7%A2%D7%94+%D7%A6%D7%99%D7%91%D7%95%D7%A8%D7%99%D7%AA&search=Array (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fd503da5f10>: Failed to establish a new connection: [Errno -5] No address associated with hostname'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 79, in _process
self.datapackage = self.process_datapackage(self.datapackage)
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 112, in process_datapackage
raise self.exc
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 60, in describe
sample = list(itertools.islice(self.iterable, self.SAMPLE_SIZE))
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 89, in handle_iterable
for x in self.iterable:
File "/datapackage_pipelines_budgetkey/pipelines/procurement/calls_for_bids/jobiz.py", line 19, in fetch_results
content = requests.get(URL.format(index)).json()
File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 677, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPSConnectionPool(host='jobiz.gov.il', port=443): Max retries exceeded with url: /ajax/results/%D7%94%D7%95%D7%93%D7%A2%D7%94%20%D7%A6%D7%99%D7%91%D7%95%D7%A8%D7%99%D7%AA/0?ie=0&typeie=%D7%94%D7%95%D7%93%D7%A2%D7%94+%D7%A6%D7%99%D7%91%D7%95%D7%A8%D7%99%D7%AA&search=Array (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fd503da5f10>: Failed to establish a new connection: [Errno -5] No address associated with hostname'))
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module>
spew_flow(flow, ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/utilities/flow_utils.py", line 46, in spew_flow
datastream = flow.datastream()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/flow.py", line 19, in datastream
return self._chain(ds)._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
[Previous line repeated 5 more times]
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 86, in _process
self.raise_exception(exception)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 96, in raise_exception
raise error from cause
dataflows.base.exceptions.ProcessorError: Errored in processor iterable_loader in position #1: HTTPSConnectionPool(host='jobiz.gov.il', port=443): Max retries exceeded with url: /ajax/results/%D7%94%D7%95%D7%93%D7%A2%D7%94%20%D7%A6%D7%99%D7%91%D7%95%D7%A8%D7%99%D7%AA/0?ie=0&typeie=%D7%94%D7%95%D7%93%D7%A2%D7%94+%D7%A6%D7%99%D7%91%D7%95%D7%A8%D7%99%D7%AA&search=Array (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fd503da5f10>: Failed to establish a new connection: [Errno -5] No address associated with hostname'))
Pipeline ID: procurement/calls_for_bids/jobiz-scraper
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/procurement/municipal/datacity-tender-fetcher.py", line 55, in flow
DF.dump_to_sql(dict(
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/to_sql.py", line 67, in __init__
with self.engine.connect():
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3295, in connect
return self._connection_cls(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 146, in __init__
Connection._handle_dbapi_exception_noconnection(
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2450, in _handle_dbapi_exception_noconnection
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 144, in __init__
self._dbapi_connection = engine.raw_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3319, in raw_connection
return self.pool.connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 448, in connect
return _ConnectionFairy._checkout(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 1272, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 712, in checkout
rec = pool._do_get()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 179, in _do_get
self._dec_overflow()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 176, in _do_get
return self._create_connection()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 389, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 674, in __init__
self.__connect()
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 901, in __connect
pool.logger.debug("Error on connect(): %s", e)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 122, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 896, in __connect
self.dbapi_connection = connection = pool._invoke_creator(self)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/create.py", line 667, in connect
return dialect.connect(*cargs_tup, **cparams)
File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 630, in connect
return self.loaded_dbapi.connect(*cargs, **cparams) # type: ignore[no-any-return] # NOQA: E501
File "/usr/local/lib/python3.9/site-packages/psycopg2/__init__.py", line 122, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) connection to server at "postgres" (10.43.75.224), port 5432 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
(Background on this error at: https://sqlalche.me/e/20/e3q8)
Pipeline ID: procurement/municipal/fetcher
dump.to_sql
ERROR :Traceback (most recent call last):
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2127, in _exec_insertmany_context
ERROR :dialect.do_execute(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 952, in do_execute
ERROR :cursor.execute(statement, parameters)
ERROR :psycopg2
ERROR :.
ERROR :OperationalError
ERROR ::
ERROR :server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
ERROR :The above exception was the direct cause of the following exception:
ERROR :Traceback (most recent call last):
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/dump/to_sql.py", line 15, in <module>
ERROR :spew_flow(flow(ctx.parameters), ctx)
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 181, in __exit__
ERROR :spew(self.datapackage, self.resource_iterator, stats=self.stats)
ERROR :File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/wrapper/wrapper.py", line 75, in spew
ERROR :for rec in res:
ERROR :File "/usr/local/lib/python3.9/site-packages/dataflows/processors/dumpers/dumper_base.py", line 69, in row_counter
ERROR :for row in iterator:
ERROR :File "/usr/local/lib/python3.9/site-packages/tableschema_sql/writer.py", line 55, in write
ERROR :for wr in self.__insert(connection):
ERROR :File "/usr/local/lib/python3.9/site-packages/tableschema_sql/writer.py", line 86, in __insert
ERROR :connection.execute(statement, self.__buffer)
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1421, in execute
ERROR :return meth(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 526, in _execute_on_connection
ERROR :return connection._execute_clauseelement(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1643, in _execute_clauseelement
ERROR :ret = self._execute_context(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1846, in _execute_context
ERROR :return self._exec_insertmany_context(dialect, context)
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2135, in _exec_insertmany_context
ERROR :self._handle_dbapi_exception(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2365, in _handle_dbapi_exception
ERROR :raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2127, in _exec_insertmany_context
ERROR :dialect.do_execute(
ERROR :File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 952, in do_execute
ERROR :cursor.execute(statement, parameters)
ERROR :sqlalchemy.exc
ERROR :.
ERROR :OperationalError
ERROR ::
ERROR :(psycopg2.OperationalError) server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
[SQL: INSERT INTO procurement_tenders_all (publication_id, publisher, publisher_unit, tender_type_he, tender_id, description, status, publication_date, last_update_date, claim_date, supplier, supplier_id, start_date, end_date, subjects, regulation, decisio ... 585058 characters truncated ... _title__999)s, %(reason__999)s, %(documents__999)s::JSONB, %(contact__999)s, %(contact_email__999)s)]
[parameters: {'supplier__0': None, 'start_date__0': None, 'publisher_unit__0': 'משרד הכלכלה - הרשות להגבלים עסקיים', 'supplier_id__0': '512615964', 'last_update_date__0': datetime.date(2013, 6, 18), 'subjects__0': 'מחשוב ותקשורת', 'publication_id__0': '543014', 'publisher__0': 'משרד הכלכלה והתעשייה', 'status__0': 'פורסם', 'volume__0': None, 'tender_type_he__0': 'פרסום כוונה להתקשרות', 'publication_date__0': datetime.date(2013, 6, 18), 'reason__0': 'תקנה 3(29) - ספק יחיד', 'documents__0': '[]', 'tender_id__0': 'none', 'source_currency__0': 'ILS', 'decision__0': 'טרם התקבלה החלטת ועדה', 'page_title__0': 'עובדי מחלקת חקירות משתמשים במכשירי PTT של חברת הוט מובייל (לשעבר מירס) אשר משמשים לצרכים מבצעים. מכשיר זה ה', 'contact_email__0': None, 'approving_level__0': 'הפטור', 'page_url__0': 'https://mr.gov.il/ilgstorefront/he/p/543014', 'description__0': 'עובדי מחלקת חקירות משתמשים במכשירי PTT של חברת הוט מובייל (לשעבר מירס) אשר משמשים לצרכים מבצעים. מכשיר זה ה', 'regulation__0': 'תקנה 3(29) - ספק יחיד', 'tender_type__0': 'exemptions', 'claim_date__0': datetime.datetime(2013, 7, 7, 0, 0), 'contact__0': None, 'end_date__0': None, 'supplier__1': None, 'start_date__1': None, 'publisher_unit__1': 'משרד החקלאות ובטחון המזון', 'supplier_id__1': '512017989', 'last_update_date__1': datetime.date(2013, 6, 18), 'subjects__1': 'שירותים אישיים ומקומיים', 'publication_id__1': '543015', 'publisher__1': 'משרד החקלאות ובטחון המזון', 'status__1': 'פורסם', 'volume__1': None, 'tender_type_he__1': 'פרסום כוונה להתקשרות', 'publication_date__1': datetime.date(2013, 6, 18), 'reason__1': 'תקנה 3(29) - ספק יחיד', 'documents__1': '[]', 'tender_id__1': 'none', 'source_currency__1': 'ILS', 'decision__1': 'טרם התקבלה החלטת ועדה', 'page_title__1': 'רכישת מוצרי מזון (כיבוד) עבור לשכת שר החקלאות', 'contact_email__1': None, 'approving_level__1': 'הפטור', 'page_url__1': 'https://mr.gov.il/ilgstorefront/he/p/543015', 'description__1': 'רכישת מוצרי מזון (כיבוד) עבור לשכת שר החקלאות', 'regulation__1': 'תקנה 3(29) - ספק יחיד' ... 26900 parameters truncated ... 'last_update_date__998': datetime.date(2013, 7, 28), 'subjects__998': 'שירותי כביסה וגיהוץ', 'publication_id__998': '544096', 'publisher__998': 'משרד הבריאות', 'status__998': 'פורסם', 'volume__998': Decimal('192240.00'), 'tender_type_he__998': 'התקשרות בפטור במכרז או בהליך תחרותי אחר', 'publication_date__998': datetime.date(2013, 7, 28), 'reason__998': 'תקנה 3(2) - התקשרות דחופה למניעת נזק של ממש', 'documents__998': '[]', 'tender_id__998': 'none', 'source_currency__998': 'ILS', 'decision__998': 'התקשרות מאושרת', 'page_title__998': 'חברת יונידרס - שרותי כביסה בבית החולים הלל יפה', 'contact_email__998': None, 'approving_level__998': 'מכרזים משרדית', 'page_url__998': 'https://mr.gov.il/ilgstorefront/he/p/544096', 'description__998': 'חברת יונידרס - שרותי כביסה בבית החולים הלל יפה', 'regulation__998': 'תקנה 3(2) - התקשרות דחופה למניעת נזק של ממש', 'tender_type__998': 'exemptions', 'claim_date__998': None, 'contact__998': None, 'end_date__998': datetime.date(2014, 1, 22), 'supplier__999': None, 'start_date__999': None, 'publisher_unit__999': 'משרד הרווחה', 'supplier_id__999': '512762584', 'last_update_date__999': datetime.date(2013, 7, 28), 'subjects__999': 'מחשוב ותקשורת', 'publication_id__999': '544097', 'publisher__999': 'משרד הרווחה', 'status__999': 'פורסם', 'volume__999': None, 'tender_type_he__999': 'פרסום כוונה להתקשרות', 'publication_date__999': datetime.date(2013, 7, 28), 'reason__999': 'תקנה 3(29) - ספק יחיד', 'documents__999': '[]', 'tender_id__999': 'none', 'source_currency__999': 'ILS', 'decision__999': 'טרם התקבלה החלטת ועדה', 'page_title__999': 'משרד הרווחה והשירותים החברתיים מעוניין להתקשר עם חברת CA בע"מ בדבר שדרוג תוכנות מתוצרתה בסביבת ה- MF תחת מערכ', 'contact_email__999': None, 'approving_level__999': 'הפטור', 'page_url__999': 'https://mr.gov.il/ilgstorefront/he/p/544097', 'description__999': 'משרד הרווחה והשירותים החברתיים מעוניין להתקשר עם חברת CA בע"מ בדבר שדרוג תוכנות מתוצרתה בסביבת ה- MF תחת מערכ', 'regulation__999': 'תקנה 3(29) - ספק יחיד', 'tender_type__999': 'exemptions', 'claim_date__999': datetime.datetime(2013, 8, 14, 0, 0), 'contact__999': None, 'end_date__999': None}]
(Background on this error at: https://sqlalche.me/e/20/e3q8)
Pipeline ID: procurement/tenders/all
sample
Traceback (most recent call last):
File "/datapackage_pipelines_budgetkey/processors/sample.py", line 11, in <module>
res_name = parameters.get('resource', datapackage['resources'][0]['name'])
IndexError: list index out of range
Pipeline ID: supports/criteria/support-criteria
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 181, in flow
DF.load(wrapper(year), format='csv',
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 21, in wrapper
return scraper(gcd, year)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 164, in scraper
return gcd.download('https://next.obudget.org/datapackages/', outfile=gcd.list_downloads()[0])
File "/datapackage_pipelines_budgetkey/common/google_chrome.py", line 204, in download
assert response.status_code == 200
AssertionError
WARNING :FAILED to teardown google-chrome
Pipeline ID: supports/supports-2023
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 181, in flow
DF.load(wrapper(year), format='csv',
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 21, in wrapper
return scraper(gcd, year)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 164, in scraper
return gcd.download('https://next.obudget.org/datapackages/', outfile=gcd.list_downloads()[0])
File "/datapackage_pipelines_budgetkey/common/google_chrome.py", line 204, in download
assert response.status_code == 200
AssertionError
WARNING :FAILED to teardown google-chrome
Pipeline ID: supports/supports-2024
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 181, in flow
DF.load(wrapper(year), format='csv',
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 21, in wrapper
return scraper(gcd, year)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 164, in scraper
return gcd.download('https://next.obudget.org/datapackages/', outfile=gcd.list_downloads()[0])
File "/datapackage_pipelines_budgetkey/common/google_chrome.py", line 204, in download
assert response.status_code == 200
AssertionError
WARNING :FAILED to teardown google-chrome
Pipeline ID: supports/supports-2025
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 13, in <module>
flow = flow_module.flow(parameters, datapackage, resources, ctx.stats)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 181, in flow
DF.load(wrapper(year), format='csv',
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 21, in wrapper
return scraper(gcd, year)
File "/datapackage_pipelines_budgetkey/pipelines/supports/scraper.py", line 164, in scraper
return gcd.download('https://next.obudget.org/datapackages/', outfile=gcd.list_downloads()[0])
File "/datapackage_pipelines_budgetkey/common/google_chrome.py", line 204, in download
assert response.status_code == 200
AssertionError
WARNING :FAILED to teardown google-chrome
Pipeline ID: supports/supports-2026
flow
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 63, in load
bytes = _RemoteStream(source, self.__http_session, self.__http_timeout).open()
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 121, in open
self.seek(0)
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 143, in seek
self.__response.raise_for_status()
File "/usr/local/lib/python3.9/site-packages/requests/models.py", line 1026, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 502 Server Error: Bad Gateway for url: https://data-input.obudget.org/api/datarecords/hierarchy
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/load.py", line 125, in process_datapackage
return self.safe_process_datapackage(dp)
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/load.py", line 186, in safe_process_datapackage
stream: Stream = Stream(self.load_source, **self.options).open()
File "/usr/local/lib/python3.9/site-packages/tabulator/stream.py", line 425, in open
self.__parser.open(source, encoding=self.__encoding)
File "/usr/local/lib/python3.9/site-packages/tabulator/parsers/json.py", line 40, in open
self.__bytes = self.__loader.load(source, mode='b', encoding=encoding)
File "/usr/local/lib/python3.9/site-packages/tabulator/loaders/remote.py", line 72, in load
raise exceptions.HTTPError(str(exception))
tabulator.exceptions.HTTPError: 502 Server Error: Bad Gateway for url: https://data-input.obudget.org/api/datarecords/hierarchy
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 79, in _process
self.datapackage = self.process_datapackage(self.datapackage)
File "/usr/local/lib/python3.9/site-packages/dataflows/processors/load.py", line 127, in process_datapackage
raise SourceLoadError('Failed to load source {!r} and options {!r}: {}'
dataflows.base.exceptions.SourceLoadError: Failed to load source 'https://data-input.obudget.org/api/datarecords/hierarchy' and options {'format': 'json', 'property': 'result', 'custom_parsers': {'xml': <class 'dataflows.processors.parsers.xml_parser.XMLParser'>, 'excel-xml': <class 'dataflows.processors.parsers.excel_xml_parser.ExcelXMLParser'>, 'sql': <class 'dataflows.processors.parsers.sql_parser.ExtendedSQLParser'>, 'geojson': <class 'dataflows.processors.parsers.geojson_parser.GeoJsonParser'>}, 'ignore_blank_headers': True, 'skip_rows': [{'type': 'preset', 'value': 'auto'}], 'headers': 1, 'sample_size': 1000}: 502 Server Error: Bad Gateway for url: https://data-input.obudget.org/api/datarecords/hierarchy
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/specs/../lib/flow.py", line 15, in <module>
spew_flow(flow, ctx)
File "/usr/local/lib/python3.9/site-packages/datapackage_pipelines/utilities/flow_utils.py", line 46, in spew_flow
datastream = flow.datastream()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/flow.py", line 19, in datastream
return self._chain(ds)._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 75, in _process
datastream = self.source._process()
[Previous line repeated 16 more times]
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 86, in _process
self.raise_exception(exception)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 97, in raise_exception
raise cause
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 79, in _process
self.datapackage = self.process_datapackage(self.datapackage)
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 112, in process_datapackage
raise self.exc
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 60, in describe
sample = list(itertools.islice(self.iterable, self.SAMPLE_SIZE))
File "/usr/local/lib/python3.9/site-packages/dataflows/helpers/iterable_loader.py", line 89, in handle_iterable
for x in self.iterable:
File "/datapackage_pipelines_budgetkey/pipelines/units/social_services/social_services.py", line 21, in all_units
hierarchies = datarecords('hierarchy')
File "/datapackage_pipelines_budgetkey/pipelines/units/social_services/social_services.py", line 7, in datarecords
DF.Flow(
File "/usr/local/lib/python3.9/site-packages/dataflows/base/flow.py", line 13, in results
return self._chain().results(on_error=on_error)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 127, in results
ds, results = self.safe_process(return_results=True, on_error=on_error)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 119, in safe_process
self.raise_exception(exception)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 97, in raise_exception
raise cause
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 102, in safe_process
ds = self._process()
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 86, in _process
self.raise_exception(exception)
File "/usr/local/lib/python3.9/site-packages/dataflows/base/datastream_processor.py", line 96, in raise_exception
raise error from cause
dataflows.base.exceptions.ProcessorError: Errored in processor load in position #1: Failed to load source 'https://data-input.obudget.org/api/datarecords/hierarchy' and options {'format': 'json', 'property': 'result', 'custom_parsers': {'xml': <class 'dataflows.processors.parsers.xml_parser.XMLParser'>, 'excel-xml': <class 'dataflows.processors.parsers.excel_xml_parser.ExcelXMLParser'>, 'sql': <class 'dataflows.processors.parsers.sql_parser.ExtendedSQLParser'>, 'geojson': <class 'dataflows.processors.parsers.geojson_parser.GeoJsonParser'>}, 'ignore_blank_headers': True, 'skip_rows': [{'type': 'preset', 'value': 'auto'}], 'headers': 1, 'sample_size': 1000}: 502 Server Error: Bad Gateway for url: https://data-input.obudget.org/api/datarecords/hierarchy
Pipeline ID: units/social_services/social_services
Pipeline ID: activities/social_services/social_service_reports
Pipeline ID: activities/social_services/social_service_suppliers
Pipeline ID: activities/social_services/social_service_tenders
Pipeline ID: activities/social_services/social_services
Pipeline ID: budget/national/changes/explanations/all
Pipeline ID: budget/national/changes/original/current-year-fixes
Pipeline ID: budget/national/changes/original/national-budget-changes
Pipeline ID: budget/national/changes/processed/national-budget-changes-aggregated
Pipeline ID: budget/national/changes/processed/transactions
We get the data the Ministry of Finance publishes in data.gov.il every year. It comes in XLS format, with one row per TAKANA and phase (original, approved, executed). In this pipeline we create from each triplet a single row that has all the data.
Pipeline ID: budget/national/original/national-budgets
This pipeline joins the different phases of the budget (allocated, revised and executed). In the original file there's a separate row for each of the phases. We like it better as a single row with all phase info. Another thing this pipeline does is to rename the column titles - to more friendly English names. Also, it create rows for all hierarchies - where upper hierarchies (2, 4 & 6 digits) are plain aggregations of the 8-digit items they contain.
Pipeline ID: budget/national/processed/aggregated-yearly
Pipeline ID: budget/national/processed/category-explanations
This pipeline joins budget items that span across years.
Pipeline ID: budget/national/processed/connected-items-explained
This pipeline joins budget items that span across years.
Pipeline ID: budget/national/processed/connected-national-budgets
Pipeline ID: budget/national/processed/just-the-total
Pipeline ID: budget/national/processed/roof-names
This pipeline joins the budget data to itself so that each item has a list of its immediate children.
Pipeline ID: budget/national/processed/with-extras
Pipeline ID: budgetkey/analysis/spending/publisher_entity_analysis
Pipeline ID: budgetkey/analysis/spending/publisher_foa_analysis
Pipeline ID: budgetkey/analysis/spending/united
Pipeline ID: budgetkey/budget-functional-aggregates
Pipeline ID: budgetkey/documents
Pipeline ID: budgetkey/elasticsearch/index_activities
Pipeline ID: budgetkey/elasticsearch/index_budget
Pipeline ID: budgetkey/elasticsearch/index_contract_spending
Pipeline ID: budgetkey/elasticsearch/index_gov_decisions
Pipeline ID: budgetkey/elasticsearch/index_muni_budgets
Pipeline ID: budgetkey/elasticsearch/index_muni_tenders
Pipeline ID: budgetkey/elasticsearch/index_reports
Pipeline ID: budgetkey/elasticsearch/index_support_programs
Pipeline ID: budgetkey/elasticsearch/index_supports
Pipeline ID: budgetkey/elasticsearch/index_tenders
Pipeline ID: budgetkey/elasticsearch/index_units
Pipeline ID: budgetkey/emails/emails
Pipeline ID: donations/candidates
Pipeline ID: donations/parties
Pipeline ID: entities/associations/guidestar/guidestar
Pipeline ID: entities/associations/guidestar/guidestar-scraper
רשימת כלל העמותות הפעילות, מתעדכן שבועית.
Pipeline ID: entities/associations/registrar/registry
Pipeline ID: entities/companies/registrar/details
רשימת אגודות שיתופיות, מתוך ממשק נתונים של מאגר של משרד הכלכלה / האגף לאיגוד שיתופי, מתעדכן שבועית
Pipeline ID: entities/cooperatives/cooperatives
Pipeline ID: entities/endowments/endowments
Pipeline ID: entities/entities
Pipeline ID: entities/fingerprints
רשימת אגודות עותומניות, מתוך תשובה לבקשת חופש מידע
Pipeline ID: entities/ottoman/ottoman-association-registry
Pipeline ID: entities/partnerships/partnerships
Pipeline ID: ezvonot/ezvonot_crontab
Pipeline ID: facilities/all/facilities_tamat
Pipeline ID: facilities/education/facilities_education
Pipeline ID: facilities/mapathataf-export/mapathataf_export
Pipeline ID: facilities/tipat-halav/facilities_tipat_halav
Pipeline ID: facilities/welfare/facilities_welfare
Pipeline ID: government_decisions/government_decisions
Pipeline ID: government_decisions/scraper
Pipeline ID: knesset/knesset_committee_decisions
Pipeline ID: lamas/municipal-data
Pipeline ID: lamas/municipality-to-city
Pipeline ID: people/association_founders/association-founders
רשימת כל המינויים והחברות מאתר כלכליסט
Pipeline ID: people/company_appointments/media/calcalist/calcalist
Pipeline ID: people/company_appointments/state_owned/correlate
Pipeline ID: people/company_appointments/state_owned/parse_pdf_resources
columns: date, url
Pipeline ID: people/company_appointments/state_owned/reports
Pipeline ID: people/company_appointments/state_owned/state-owned-nominations
Pipeline ID: people/municipal_appointments/municipal_representitives
Pipeline ID: people/political_donations/political_donations
Pipeline ID: people/procurement/procurement-individuals
Pipeline ID: procurement/calls_for_bids/calls-for-bids-moh
אנו סוכמים מידע מתוך הדוחות הרבעוניים לתוך טבלה המכילה את המידע העדכני ביותר.
Pipeline ID: procurement/spending/latest-contract-spending
Pipeline ID: procurement/spending/quarterly-contract-spending-report-uris
כל משרדי הממשלה ויחידות הסמך (שמחוברות למערכת מרכבה) צריכים לפרסם אחת לרבעון (תוך 45 יום מסוף הרבעון) דו״ח התקשרויות לפי פורמט מוגדר היטב הממומש כדו״ח מובנה במערכת המרכבה. כל המשרדים מדווחים עצמאית והדו״חות עולים לאתר היחידה לחופש המידע, משם אנו אוספים אותם. התוצר של השרשר הזה הוא פירוט כל הדיווחים של כל המשרדים.
Pipeline ID: procurement/spending/quarterly-contract-spending-reports
Pipeline ID: procurement/spending/quarterly-contract-spending-reports-data
Pipeline ID: procurement/spending/spending-by-entity
Pipeline ID: procurement/tenders/processed
Pipeline ID: reports/all
Pipeline ID: reports/ngos/ngo-activity-report
Pipeline ID: reports/ngos/ngo-district-report
Pipeline ID: simpledb/process_simple_dbs
Pipeline ID: support_programs/all-support-programs
Pipeline ID: supports/by-payment-year
Pipeline ID: supports/by-request-year
Pipeline ID: supports/criteria/support-criteria-doj
Pipeline ID: supports/supports-2004
Pipeline ID: supports/supports-2005
Pipeline ID: supports/supports-2006
Pipeline ID: supports/supports-2007
Pipeline ID: supports/supports-2008
Pipeline ID: supports/supports-2009
Pipeline ID: supports/supports-2010
Pipeline ID: supports/supports-2011
Pipeline ID: supports/supports-2012
Pipeline ID: supports/supports-2013
Pipeline ID: supports/supports-2014
Pipeline ID: supports/supports-2015
Pipeline ID: supports/supports-2016
Pipeline ID: supports/supports-2017
Pipeline ID: supports/supports-2018
Pipeline ID: supports/supports-2019
Pipeline ID: supports/supports-2020
Pipeline ID: supports/supports-2021
Pipeline ID: supports/supports-2022
Pipeline ID: supports/supports-by-entity
Pipeline ID: supports/with-entities
Pipeline ID: supports/with-keys