[2023-10-04 20:01:13,935: INFO/MainProcess] Connected to redis://:**@127.0.0.1:6379/0 [2023-10-04 20:01:13,944: INFO/MainProcess] mingle: searching for neighbors [2023-10-04 20:01:14,961: INFO/MainProcess] mingle: sync with 1 nodes [2023-10-04 20:01:14,962: INFO/MainProcess] mingle: sync complete [2023-10-04 20:01:14,979: INFO/MainProcess] worker2@vps.infomin.solutions ready. [2023-10-04 20:01:49,439: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[b4374669-53e2-42ec-aceb-b8683b9a26cb] received [2023-10-04 20:04:21,119: ERROR/MainProcess] Pool callback raised exception: OperationalError('(sqlite3.OperationalError) attempt to write a readonly database') Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1965, in _exec_single_context self.dialect.do_execute( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 921, in do_execute cursor.execute(statement, parameters) sqlite3.OperationalError: attempt to write a readonly database The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/billiard/pool.py", line 1793, in safe_apply_callback fun(*args, **kwargs) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/request.py", line 624, in on_failure self.task.backend.mark_as_failure( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/backends/base.py", line 167, in mark_as_failure self.store_result(task_id, exc, state, File "/home/infominsolutions/scheduler_infomin_solutions/backend.py", line 22, in store_result raise e File "/home/infominsolutions/scheduler_infomin_solutions/backend.py", line 19, in store_result session.commit() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1923, in commit trans.commit(_to_root=True) File "", line 2, in commit File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/state_changes.py", line 139, in _go ret_value = fn(self, *arg, **kw) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1239, in commit self._prepare_impl() File "", line 2, in _prepare_impl File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/state_changes.py", line 139, in _go ret_value = fn(self, *arg, **kw) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1214, in _prepare_impl self.session.flush() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 4179, in flush self._flush(objects) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 4315, in _flush transaction.rollback(_capture_exception=True) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/util/langhelpers.py", line 146, in __exit__ raise exc_value.with_traceback(exc_tb) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 4275, in _flush flush_context.execute() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/unitofwork.py", line 466, in execute rec.execute(self) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/unitofwork.py", line 642, in execute util.preloaded.orm_persistence.save_obj( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/persistence.py", line 93, in save_obj _emit_insert_statements( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/persistence.py", line 1226, in _emit_insert_statements result = connection.execute( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1412, in execute return meth( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 516, in _execute_on_connection return connection._execute_clauseelement( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1635, in _execute_clauseelement ret = self._execute_context( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1844, in _execute_context return self._exec_single_context( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1984, in _exec_single_context self._handle_dbapi_exception( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 2339, in _handle_dbapi_exception raise sqlalchemy_exception.with_traceback(exc_info[2]) from e File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1965, in _exec_single_context self.dialect.do_execute( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 921, in do_execute cursor.execute(statement, parameters) sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) attempt to write a readonly database [SQL: INSERT INTO task (task_id, result, state, api_account_id) VALUES (?, ?, ?, ?)] [parameters: ('b4374669-53e2-42ec-aceb-b8683b9a26cb', '(sqlite3.OperationalError) attempt to write a readonly database\n[SQL: INSERT INTO task (task_id, result, state, api_account_id) VALUES (?, ?, ?, ?)] ... (332 characters truncated) ... , 1)]\\n(Background on this error at: https://sqlalche.me/e/20/e3q8)", \'FAILURE\', 1)]\n(Background on this error at: https://sqlalche.me/e/20/e3q8)', 'FAILURE', 1)] (Background on this error at: https://sqlalche.me/e/20/e3q8) [2023-10-05 01:03:59,989: INFO/MainProcess] Connected to redis://:**@127.0.0.1:6379/0 [2023-10-05 01:03:59,999: INFO/MainProcess] mingle: searching for neighbors [2023-10-05 01:04:01,026: INFO/MainProcess] mingle: sync with 1 nodes [2023-10-05 01:04:01,028: INFO/MainProcess] mingle: sync complete [2023-10-05 01:04:01,052: INFO/MainProcess] worker2@vps.infomin.solutions ready. [2023-10-05 01:08:49,692: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[b4d34adb-fa84-4491-95c8-de5a4ad27299] received [2023-10-05 01:10:40,155: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[6d422ed6-96b7-4029-a8ea-42cc626b3b95] received [2023-10-18 19:10:04,878: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-10-21 19:30:42,556: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-10-22 19:04:48,389: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-10-22 19:06:24,591: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-10-22 19:06:24,720: WARNING/MainProcess] Substantial drift from worker1@vps.infomin.solutions may mean clocks are out of sync. Current drift is 33 seconds. [orig: 2023-10-22 19:06:24.706784 recv: 2023-10-22 19:05:51.314591] [2023-10-22 19:06:32,651: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-10-22 19:11:57,915: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-10-22 19:12:28,370: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-10-24 14:19:32,702: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[91bf2568-c172-46f4-ab56-f6fbeaf5bf2b] received [2023-10-24 14:20:19,685: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[9c0e4bbe-e0d9-440d-adc7-1bbcc55ad140] received [2023-10-24 14:25:09,117: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[320cbb9f-58b7-4dcd-92ae-72944a8d6ea1] received [2023-10-24 14:25:11,957: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[f527c6e9-87df-424d-b34d-87ae97a37670] received [2023-10-24 14:25:26,184: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[06040a3e-a562-41dc-b2f1-f961c3ab4641] received [2023-10-24 14:25:36,065: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[cfd454b9-1aee-4da2-b4d7-c497160ded06] received [2023-10-24 14:25:44,822: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[8936daba-423c-4663-bf04-70393843e340] received [2023-11-01 13:47:19,064: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-11-09 19:20:12,111: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-11-16 18:57:54,406: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-11-17 19:19:27,508: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-11-25 17:09:17,557: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-11-27 19:29:52,145: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-11-29 18:50:32,671: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-12-01 19:30:06,517: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-12-08 19:26:04,838: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-12-09 18:53:05,425: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-12-13 19:21:02,934: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-12-16 18:50:34,411: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-12-19 19:21:50,686: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2023-12-25 15:20:32,673: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[e11d9014-63b4-466c-bcc2-90fedcffbda3] received [2023-12-27 19:30:22,379: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-01-02 02:23:34,973: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-01-02 02:24:15,647: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-01-02 02:25:21,998: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-01-02 02:26:04,766: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-01-02 02:26:10,811: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-01-02 02:26:17,105: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-01-02 02:27:09,348: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-01-18 19:22:52,615: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-01-20 19:11:28,038: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-01-25 17:44:34,777: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-02-03 17:03:01,688: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 2.00 seconds... (1/100) [2024-02-03 17:03:03,710: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 4.00 seconds... (2/100) [2024-02-03 17:03:07,724: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 6.00 seconds... (3/100) [2024-02-03 17:03:13,775: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 8.00 seconds... (4/100) [2024-02-03 17:03:21,789: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 10.00 seconds... (5/100) [2024-02-03 17:03:31,819: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 12.00 seconds... (6/100) [2024-02-03 17:03:43,858: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 14.00 seconds... (7/100) [2024-02-03 17:03:57,881: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 16.00 seconds... (8/100) [2024-02-03 17:04:13,937: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 18.00 seconds... (9/100) [2024-02-03 17:04:31,961: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 20.00 seconds... (10/100) [2024-02-03 17:04:51,983: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 22.00 seconds... (11/100) [2024-02-03 17:05:14,010: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 24.00 seconds... (12/100) [2024-02-03 17:05:38,059: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 26.00 seconds... (13/100) [2024-02-03 17:06:04,094: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 28.00 seconds... (14/100) [2024-02-03 17:06:32,128: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 30.00 seconds... (15/100) [2024-02-03 17:07:02,173: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:07:34,214: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:08:06,257: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:08:38,297: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:09:10,347: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:09:42,389: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:10:14,434: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:10:46,471: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:11:18,511: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:11:50,547: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:12:22,587: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:12:54,624: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:13:26,665: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:13:58,707: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:14:30,744: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:15:02,783: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:15:34,827: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:16:06,866: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:16:38,905: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:17:10,943: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:17:42,981: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:18:15,022: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:18:47,064: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:19:19,100: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:19:51,135: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:20:23,175: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:20:55,211: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:21:27,251: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:21:59,290: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:22:31,332: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:23:03,369: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:23:35,405: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:24:07,444: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:24:39,481: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:25:11,519: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:25:43,556: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:26:15,593: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:26:47,629: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:27:19,667: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:27:51,713: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:28:23,749: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:28:55,787: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:29:27,827: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:29:59,863: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:30:31,904: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:31:03,940: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:31:35,975: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:32:08,013: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:32:40,048: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:33:12,087: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:33:44,123: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:34:16,163: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:34:48,200: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:35:20,241: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:35:52,278: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:36:24,314: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:36:56,349: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:37:28,387: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:38:00,425: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:38:32,459: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:39:04,501: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:39:36,542: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:40:08,588: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:40:40,626: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:41:12,662: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:41:44,703: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:42:16,740: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:42:48,778: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:43:20,815: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:43:52,851: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:44:24,894: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:44:56,932: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:45:28,978: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:46:01,014: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:46:33,052: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:47:05,088: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:47:37,125: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:48:09,163: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:48:41,200: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:49:13,237: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:49:45,274: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:50:17,317: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:50:49,357: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:51:21,392: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:51:53,427: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2024-02-03 17:52:25,468: CRITICAL/MainProcess] Unrecoverable error: OperationalError('Error 111 connecting to 127.0.0.1:6379. Connection refused.') Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/transport/virtual/base.py", line 951, in create_channel return self._avail_channels.pop() IndexError: pop from empty list During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 264, in connect sock = self.retry.call_with_retry( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/retry.py", line 46, in call_with_retry return do() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 265, in lambda: self._connect(), lambda error: self.disconnect(error) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 627, in _connect raise err File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 615, in _connect sock.connect(socket_address) ConnectionRefusedError: [Errno 111] Connection refused During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 472, in _reraise_as_library_errors yield File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 459, in _ensure_connection return retry_over_time( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/utils/functional.py", line 318, in retry_over_time return fun(*args, **kwargs) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 934, in _connection_factory self._connection = self._establish_connection() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 860, in _establish_connection conn = self.transport.establish_connection() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/transport/virtual/base.py", line 975, in establish_connection self._avail_channels.append(self.create_channel(self)) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/transport/virtual/base.py", line 953, in create_channel channel = self.Channel(connection) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/transport/redis.py", line 741, in __init__ self.client.ping() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/commands/core.py", line 1216, in ping return self.execute_command("PING", **kwargs) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/client.py", line 533, in execute_command conn = self.connection or pool.get_connection(command_name, **options) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 1086, in get_connection connection.connect() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 270, in connect raise ConnectionError(self._error_message(e)) redis.exceptions.ConnectionError: Error 111 connecting to 127.0.0.1:6379. Connection refused. The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/worker.py", line 202, in start self.blueprint.start(self) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/bootsteps.py", line 116, in start step.start(parent) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/bootsteps.py", line 365, in start return self.obj.start() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/consumer/consumer.py", line 340, in start blueprint.start(self) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/bootsteps.py", line 116, in start step.start(parent) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/consumer/connection.py", line 21, in start c.connection = c.connect() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/consumer/consumer.py", line 469, in connect conn = self.connection_for_read(heartbeat=self.amqheartbeat) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/consumer/consumer.py", line 475, in connection_for_read return self.ensure_connected( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/consumer/consumer.py", line 526, in ensure_connected conn = conn.ensure_connection( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 406, in ensure_connection self._ensure_connection(*args, **kwargs) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 459, in _ensure_connection return retry_over_time( File "/usr/lib64/python3.8/contextlib.py", line 131, in __exit__ self.gen.throw(type, value, traceback) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 476, in _reraise_as_library_errors raise ConnectionError(str(exc)) from exc kombu.exceptions.OperationalError: Error 111 connecting to 127.0.0.1:6379. Connection refused. [2024-06-12 21:17:20,381: INFO/MainProcess] Connected to redis://:**@127.0.0.1:6379/0 [2024-06-12 21:17:20,391: INFO/MainProcess] mingle: searching for neighbors [2024-06-12 21:17:21,422: INFO/MainProcess] mingle: sync with 1 nodes [2024-06-12 21:17:21,424: INFO/MainProcess] mingle: sync complete [2024-06-12 21:17:21,468: INFO/MainProcess] worker2@vps.infomin.solutions ready. [2024-06-12 21:17:32,090: INFO/MainProcess] Events of group {task} enabled by remote. [2024-06-12 21:18:32,201: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[770e28e5-0d06-404a-82de-f2bc7d1c3580] received [2024-06-12 21:22:55,877: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[37bac59e-5c76-4014-9c29-0a235c82b20d] received [2024-06-12 21:24:58,092: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[2c78e221-2606-412a-92ba-9f14384967c4] received [2024-06-12 23:38:51,635: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[3bb4108e-b087-4b1b-934f-fbc99cf596a3] received [2024-06-12 23:51:21,556: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[c2c638fa-5682-45c4-a6e3-2c15a3c3db0b] received [2024-06-12 23:53:21,565: INFO/MainProcess] Terminating c2c638fa-5682-45c4-a6e3-2c15a3c3db0b (Signals.SIGTERM) [2024-06-12 23:53:21,846: ERROR/MainProcess] pidbox command error: OperationalError('(mysql.connector.errors.OperationalError) MySQL Connection not available.') Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1810, in _execute_context context = constructor( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 1398, in _init_compiled self.cursor = self.create_cursor() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 1722, in create_cursor return self.create_default_cursor() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 1728, in create_default_cursor return self._dbapi_connection.cursor() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 1489, in cursor return self.dbapi_connection.cursor(*args, **kwargs) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/mysql/connector/connection_cext.py", line 700, in cursor raise OperationalError("MySQL Connection not available.") mysql.connector.errors.OperationalError: MySQL Connection not available. The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/pidbox.py", line 102, in dispatch reply = handle(method, arguments) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/pidbox.py", line 124, in handle_cast return self.handle(method, arguments) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/pidbox.py", line 118, in handle return self.handlers[method](self.state, **arguments) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/control.py", line 149, in revoke task_ids = _revoke(state, task_ids, terminate, signal, **kwargs) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/control.py", line 224, in _revoke request.terminate(state.consumer.pool, signal=signum) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/request.py", line 417, in terminate self._announce_revoked('terminated', True, signal, False) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/request.py", line 453, in _announce_revoked self.task.backend.mark_as_revoked( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/backends/base.py", line 267, in mark_as_revoked self.store_result(task_id, exc, state, File "/home/infominsolutions/scheduler_infomin_solutions/backend.py", line 22, in store_result raise e File "/home/infominsolutions/scheduler_infomin_solutions/backend.py", line 16, in store_result api_account = session.query(connection.ApiAccount).filter_by(api_key=api_key).first() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 2743, in first return self.limit(1)._iter().first() # type: ignore File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 2842, in _iter result: Union[ScalarResult[_T], Result[_T]] = self.session.execute( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 2262, in execute return self._execute_internal( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 2144, in _execute_internal result: Result[Any] = compile_state_cls.orm_execute_statement( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/orm/context.py", line 293, in orm_execute_statement result = conn.execute( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1412, in execute return meth( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 516, in _execute_on_connection return connection._execute_clauseelement( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1635, in _execute_clauseelement ret = self._execute_context( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1816, in _execute_context self._handle_dbapi_exception( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 2339, in _handle_dbapi_exception raise sqlalchemy_exception.with_traceback(exc_info[2]) from e File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1810, in _execute_context context = constructor( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 1398, in _init_compiled self.cursor = self.create_cursor() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 1722, in create_cursor return self.create_default_cursor() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 1728, in create_default_cursor return self._dbapi_connection.cursor() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 1489, in cursor return self.dbapi_connection.cursor(*args, **kwargs) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/mysql/connector/connection_cext.py", line 700, in cursor raise OperationalError("MySQL Connection not available.") sqlalchemy.exc.OperationalError: (mysql.connector.errors.OperationalError) MySQL Connection not available. [SQL: SELECT api_account.id AS api_account_id, api_account.api_key AS api_account_api_key FROM api_account WHERE api_account.api_key = %(api_key_1)s LIMIT %(param_1)s] [parameters: [{}]] (Background on this error at: https://sqlalche.me/e/20/e3q8) [2024-06-16 17:02:57,784: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-06-23 02:44:59,488: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[e2ec3a29-5759-4c04-8c08-2fd75f3186fb] received [2024-06-23 02:46:56,790: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[34e116d7-136f-485b-be51-a81b0fff97b9] received [2024-06-23 02:53:56,938: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[bf90ac89-80c5-47bd-aab3-1d0dd41d50cd] received [2024-06-24 13:03:51,640: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[bac1cd8b-8b2f-41b9-a84b-cf76d264febe] received [2024-06-24 18:06:50,196: INFO/MainProcess] Connected to redis://:**@127.0.0.1:6379/0 [2024-06-24 18:06:50,207: INFO/MainProcess] mingle: searching for neighbors [2024-06-24 18:06:51,237: INFO/MainProcess] mingle: sync with 1 nodes [2024-06-24 18:06:51,238: INFO/MainProcess] mingle: sync complete [2024-06-24 18:06:51,269: INFO/MainProcess] worker2@vps.infomin.solutions ready. [2024-06-24 18:06:54,891: INFO/MainProcess] Events of group {task} enabled by remote. [2024-07-02 15:26:00,739: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-06 03:46:23,738: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-06 03:46:23,929: WARNING/MainProcess] Substantial drift from worker1@vps.infomin.solutions may mean clocks are out of sync. Current drift is 39 seconds. [orig: 2024-07-06 03:46:23.893183 recv: 2024-07-06 03:45:44.704107] [2024-07-06 03:47:19,963: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-06 03:47:53,027: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-06 03:48:18,863: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-09 23:50:40,794: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-09 23:51:06,073: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-09 23:51:21,247: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-09 23:51:28,078: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-09 23:51:46,675: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-09 23:51:57,414: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-09 23:52:04,482: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-10 00:45:45,057: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-11 06:05:27,009: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-12 01:31:16,677: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-12 07:13:37,954: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-12 07:14:04,602: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-12 07:14:16,713: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-12 07:14:34,655: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-12 07:14:41,840: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-12 07:15:01,044: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-12 07:38:03,633: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-12 07:38:08,891: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-12 07:38:20,346: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-12 15:01:16,023: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-12 15:01:36,368: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-12 15:01:50,845: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-14 07:04:10,521: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-14 07:04:39,267: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-07-16 07:47:38,304: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-08-07 10:45:26,861: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-08-07 10:45:54,800: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-08-07 10:46:35,753: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-08-16 06:56:34,668: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-08-16 06:57:07,014: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-08-21 23:57:55,200: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-08-22 10:33:16,625: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-08-22 10:33:51,035: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-08-22 10:38:41,180: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-08-22 22:51:24,216: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-08-23 23:56:24,790: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-08-24 03:28:53,475: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-08-24 04:19:45,947: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-09-04 17:17:01,394: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-09-14 20:53:21,050: INFO/MainProcess] missed heartbeat from worker1@vps.infomin.solutions [2024-09-18 15:07:44,436: INFO/MainProcess] Task scheduler_infomin_solutions.app.trigger_webhook[8c7001e8-bdeb-464d-bb03-4902d5777df5] received [2025-03-23 12:15:30,004: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 2.00 seconds... (1/100) [2025-03-23 12:15:32,017: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 4.00 seconds... (2/100) [2025-03-23 12:15:36,026: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 6.00 seconds... (3/100) [2025-03-23 12:15:42,042: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 8.00 seconds... (4/100) [2025-03-23 12:15:50,053: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 10.00 seconds... (5/100) [2025-03-23 12:16:00,067: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 12.00 seconds... (6/100) [2025-03-23 12:16:12,083: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 14.00 seconds... (7/100) [2025-03-23 12:16:26,107: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 16.00 seconds... (8/100) [2025-03-23 12:16:42,124: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 18.00 seconds... (9/100) [2025-03-23 12:17:00,144: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 20.00 seconds... (10/100) [2025-03-23 12:17:20,168: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 22.00 seconds... (11/100) [2025-03-23 12:17:42,192: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 24.00 seconds... (12/100) [2025-03-23 12:18:06,224: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 26.00 seconds... (13/100) [2025-03-23 12:18:32,252: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 28.00 seconds... (14/100) [2025-03-23 12:19:00,296: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 30.00 seconds... (15/100) [2025-03-23 12:19:30,334: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:20:02,378: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:20:34,414: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:21:06,451: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:21:38,488: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:22:10,523: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:22:42,558: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:23:14,595: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:23:46,631: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:24:18,666: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:24:50,701: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:25:22,736: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:25:54,771: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:26:26,814: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:26:58,852: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:27:30,886: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:28:02,922: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:28:34,961: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:29:06,995: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:29:39,032: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:30:11,068: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:30:43,104: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:31:15,140: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:31:47,176: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:32:19,216: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:32:51,252: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:33:23,289: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:33:55,325: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:34:27,360: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:34:59,397: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:35:31,434: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:36:03,471: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:36:35,507: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:37:07,545: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:37:39,584: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:38:11,620: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:38:43,659: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:39:15,696: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:39:47,733: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:40:19,767: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:40:51,804: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:41:23,839: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:41:55,877: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:42:27,912: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:42:59,949: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:43:31,985: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:44:04,041: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:44:36,076: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:45:08,111: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:45:40,148: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:46:12,183: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:46:44,218: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:47:16,254: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:47:48,290: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:48:20,325: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:48:52,365: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:49:24,398: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:49:56,434: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:50:28,472: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:51:00,511: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:51:32,549: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:52:04,582: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:52:36,621: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:53:08,657: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:53:40,694: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:54:12,729: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:54:44,765: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:55:16,808: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:55:48,846: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:56:20,881: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:56:52,917: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:57:24,951: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:57:56,985: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:58:29,015: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:59:01,049: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 12:59:33,085: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 13:00:05,133: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 13:00:37,173: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 13:01:09,212: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 13:01:41,248: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 13:02:13,284: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 13:02:45,320: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 13:03:17,357: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 13:03:49,394: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 13:04:21,431: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-03-23 13:04:53,467: CRITICAL/MainProcess] Unrecoverable error: OperationalError('Error 111 connecting to 127.0.0.1:6379. Connection refused.') Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/transport/virtual/base.py", line 951, in create_channel return self._avail_channels.pop() IndexError: pop from empty list During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 264, in connect sock = self.retry.call_with_retry( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/retry.py", line 46, in call_with_retry return do() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 265, in lambda: self._connect(), lambda error: self.disconnect(error) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 627, in _connect raise err File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 615, in _connect sock.connect(socket_address) ConnectionRefusedError: [Errno 111] Connection refused During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 472, in _reraise_as_library_errors yield File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 459, in _ensure_connection return retry_over_time( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/utils/functional.py", line 318, in retry_over_time return fun(*args, **kwargs) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 934, in _connection_factory self._connection = self._establish_connection() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 860, in _establish_connection conn = self.transport.establish_connection() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/transport/virtual/base.py", line 975, in establish_connection self._avail_channels.append(self.create_channel(self)) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/transport/virtual/base.py", line 953, in create_channel channel = self.Channel(connection) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/transport/redis.py", line 741, in __init__ self.client.ping() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/commands/core.py", line 1216, in ping return self.execute_command("PING", **kwargs) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/client.py", line 533, in execute_command conn = self.connection or pool.get_connection(command_name, **options) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 1086, in get_connection connection.connect() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 270, in connect raise ConnectionError(self._error_message(e)) redis.exceptions.ConnectionError: Error 111 connecting to 127.0.0.1:6379. Connection refused. The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/worker.py", line 202, in start self.blueprint.start(self) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/bootsteps.py", line 116, in start step.start(parent) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/bootsteps.py", line 365, in start return self.obj.start() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/consumer/consumer.py", line 340, in start blueprint.start(self) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/bootsteps.py", line 116, in start step.start(parent) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/consumer/connection.py", line 21, in start c.connection = c.connect() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/consumer/consumer.py", line 469, in connect conn = self.connection_for_read(heartbeat=self.amqheartbeat) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/consumer/consumer.py", line 475, in connection_for_read return self.ensure_connected( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/consumer/consumer.py", line 526, in ensure_connected conn = conn.ensure_connection( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 406, in ensure_connection self._ensure_connection(*args, **kwargs) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 459, in _ensure_connection return retry_over_time( File "/usr/lib64/python3.8/contextlib.py", line 131, in __exit__ self.gen.throw(type, value, traceback) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 476, in _reraise_as_library_errors raise ConnectionError(str(exc)) from exc kombu.exceptions.OperationalError: Error 111 connecting to 127.0.0.1:6379. Connection refused. [2025-04-10 17:06:39,161: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 2.00 seconds... (1/100) [2025-04-10 17:06:41,165: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 4.00 seconds... (2/100) [2025-04-10 17:06:45,177: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 6.00 seconds... (3/100) [2025-04-10 17:06:51,188: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 8.00 seconds... (4/100) [2025-04-10 17:06:59,200: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 10.00 seconds... (5/100) [2025-04-10 17:07:09,212: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 12.00 seconds... (6/100) [2025-04-10 17:07:21,227: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 14.00 seconds... (7/100) [2025-04-10 17:07:35,282: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 16.00 seconds... (8/100) [2025-04-10 17:07:51,302: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 18.00 seconds... (9/100) [2025-04-10 17:08:09,328: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 20.00 seconds... (10/100) [2025-04-10 17:08:29,353: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 22.00 seconds... (11/100) [2025-04-10 17:08:51,384: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 24.00 seconds... (12/100) [2025-04-10 17:09:15,414: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 26.00 seconds... (13/100) [2025-04-10 17:09:41,449: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 28.00 seconds... (14/100) [2025-04-10 17:10:09,481: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 30.00 seconds... (15/100) [2025-04-10 17:10:39,516: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:11:11,554: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:11:43,590: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:12:15,626: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:12:47,663: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:13:19,704: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:13:51,743: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:14:23,779: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:14:55,823: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:15:27,867: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:15:59,904: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:16:31,942: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:17:03,980: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:17:36,019: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:18:08,056: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:18:40,091: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:19:12,127: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:19:44,162: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:20:16,201: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:20:48,246: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:21:20,283: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:21:52,320: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:22:24,355: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:22:56,391: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:23:28,429: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:24:00,464: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:24:32,501: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:25:04,537: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:25:36,574: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:26:08,613: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:26:40,657: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:27:12,695: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:27:44,730: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:28:16,768: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:28:48,805: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:29:20,843: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:29:52,878: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:30:24,918: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:30:56,954: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:31:29,002: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:32:01,057: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:32:33,102: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:33:05,162: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:33:37,199: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:34:09,238: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:34:41,275: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:35:13,312: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:35:45,351: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:36:17,392: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:36:49,436: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:37:21,472: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:37:53,525: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:38:25,562: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:38:57,599: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:39:29,641: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:40:01,686: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:40:33,728: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:41:05,766: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:41:37,801: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:42:09,841: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:42:41,878: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:43:13,913: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:43:45,951: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:44:17,989: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:44:50,025: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:45:22,068: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:45:54,107: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:46:26,143: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:46:58,179: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:47:30,213: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:48:02,251: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:48:34,286: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:49:06,321: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:49:38,357: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:50:10,400: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:50:42,435: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:51:14,471: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:51:46,508: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:52:18,545: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:52:50,581: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:53:22,623: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:53:54,666: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:54:26,702: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:54:58,738: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:55:30,782: ERROR/MainProcess] consumer: Cannot connect to redis://:**@127.0.0.1:6379/0: Error 111 connecting to 127.0.0.1:6379. Connection refused.. Trying again in 32.00 seconds... (16/100) [2025-04-10 17:56:02,818: CRITICAL/MainProcess] Unrecoverable error: OperationalError('Error 111 connecting to 127.0.0.1:6379. Connection refused.') Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/transport/virtual/base.py", line 951, in create_channel return self._avail_channels.pop() IndexError: pop from empty list During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 264, in connect sock = self.retry.call_with_retry( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/retry.py", line 46, in call_with_retry return do() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 265, in lambda: self._connect(), lambda error: self.disconnect(error) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 627, in _connect raise err File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 615, in _connect sock.connect(socket_address) ConnectionRefusedError: [Errno 111] Connection refused During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 472, in _reraise_as_library_errors yield File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 459, in _ensure_connection return retry_over_time( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/utils/functional.py", line 318, in retry_over_time return fun(*args, **kwargs) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 934, in _connection_factory self._connection = self._establish_connection() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 860, in _establish_connection conn = self.transport.establish_connection() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/transport/virtual/base.py", line 975, in establish_connection self._avail_channels.append(self.create_channel(self)) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/transport/virtual/base.py", line 953, in create_channel channel = self.Channel(connection) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/transport/redis.py", line 741, in __init__ self.client.ping() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/commands/core.py", line 1216, in ping return self.execute_command("PING", **kwargs) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/client.py", line 533, in execute_command conn = self.connection or pool.get_connection(command_name, **options) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 1086, in get_connection connection.connect() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/redis/connection.py", line 270, in connect raise ConnectionError(self._error_message(e)) redis.exceptions.ConnectionError: Error 111 connecting to 127.0.0.1:6379. Connection refused. The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/worker.py", line 202, in start self.blueprint.start(self) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/bootsteps.py", line 116, in start step.start(parent) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/bootsteps.py", line 365, in start return self.obj.start() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/consumer/consumer.py", line 340, in start blueprint.start(self) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/bootsteps.py", line 116, in start step.start(parent) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/consumer/connection.py", line 21, in start c.connection = c.connect() File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/consumer/consumer.py", line 469, in connect conn = self.connection_for_read(heartbeat=self.amqheartbeat) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/consumer/consumer.py", line 475, in connection_for_read return self.ensure_connected( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/celery/worker/consumer/consumer.py", line 526, in ensure_connected conn = conn.ensure_connection( File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 406, in ensure_connection self._ensure_connection(*args, **kwargs) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 459, in _ensure_connection return retry_over_time( File "/usr/lib64/python3.8/contextlib.py", line 131, in __exit__ self.gen.throw(type, value, traceback) File "/home/infominsolutions/scheduler_infomin_solutions/venv/lib/python3.8/site-packages/kombu/connection.py", line 476, in _reraise_as_library_errors raise ConnectionError(str(exc)) from exc kombu.exceptions.OperationalError: Error 111 connecting to 127.0.0.1:6379. Connection refused.