diff --git a/pandaharvester/commit_timestamp.py b/pandaharvester/commit_timestamp.py index 74441a4d..2eaec30a 100644 --- a/pandaharvester/commit_timestamp.py +++ b/pandaharvester/commit_timestamp.py @@ -1 +1 @@ -timestamp = "04-03-2024 13:34:47 on flin (by mightqxc)" +timestamp = "07-03-2024 11:35:36 on flin (by mightqxc)" diff --git a/pandaharvester/harvesterbody/cacher.py b/pandaharvester/harvesterbody/cacher.py index 939939fa..3d1cc974 100644 --- a/pandaharvester/harvesterbody/cacher.py +++ b/pandaharvester/harvesterbody/cacher.py @@ -7,6 +7,7 @@ import requests import requests.exceptions + from pandaharvester.harvesterbody.agent_base import AgentBase from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils @@ -40,7 +41,7 @@ def execute(self, force_update=False, skip_lock=False, n_thread=0): locked = self.dbProxy.get_process_lock("cacher", self.get_pid(), harvester_config.cacher.sleepTime) if locked or skip_lock: mainLog.debug("getting information") - timeLimit = datetime.datetime.utcnow() - datetime.timedelta(minutes=harvester_config.cacher.refreshInterval) + timeLimit = core_utils.naive_utcnow() - datetime.timedelta(minutes=harvester_config.cacher.refreshInterval) itemsList = [] nItems = 4 for tmpStr in harvester_config.cacher.data: diff --git a/pandaharvester/harvesterbody/command_manager.py b/pandaharvester/harvesterbody/command_manager.py index 1a5e7264..81688af6 100644 --- a/pandaharvester/harvesterbody/command_manager.py +++ b/pandaharvester/harvesterbody/command_manager.py @@ -1,7 +1,6 @@ import datetime import socket -from future.utils import iteritems from pandaharvester import commit_timestamp, panda_pkg_info from pandaharvester.harvesterbody.agent_base import AgentBase from pandaharvester.harvesterconfig import harvester_config @@ -36,7 +35,7 @@ def convert_to_command_specs(self, commands): for command in commands: command_spec = CommandSpec() command_spec.convert_command_json(command) - for comStr, receiver in iteritems(CommandSpec.receiver_map): + for comStr, receiver in CommandSpec.receiver_map.items(): if command_spec.command.startswith(comStr): command_spec.receiver = receiver break @@ -55,7 +54,7 @@ def run(self): # send command list to be received siteNames = set() commandList = [] - for queueName, queueConfig in iteritems(self.queueConfigMapper.get_active_queues()): + for queueName, queueConfig in self.queueConfigMapper.get_active_queues().items(): if queueConfig is None or queueConfig.runMode != "slave": continue # one command for all queues in one site @@ -70,7 +69,7 @@ def run(self): # one command for each queue commandItem = {"command": CommandSpec.COM_setNWorkers, "computingSite": queueConfig.siteName, "resourceType": queueConfig.resourceType} commandList.append(commandItem) - data = {"startTime": datetime.datetime.utcnow(), "sw_version": panda_pkg_info.release_version, "commit_stamp": commit_timestamp.timestamp} + data = {"startTime": core_utils.naive_utcnow(), "sw_version": panda_pkg_info.release_version, "commit_stamp": commit_timestamp.timestamp} if len(commandList) > 0: main_log.debug("sending command list to receive") data["commands"] = commandList @@ -84,8 +83,8 @@ def run(self): main_log.debug("polling commands loop") # send heartbeat - if self.lastHeartbeat is None or self.lastHeartbeat < datetime.datetime.utcnow() - datetime.timedelta(minutes=10): - self.lastHeartbeat = datetime.datetime.utcnow() + if self.lastHeartbeat is None or self.lastHeartbeat < core_utils.naive_utcnow() - datetime.timedelta(minutes=10): + self.lastHeartbeat = core_utils.naive_utcnow() self.communicator.is_alive({}) continuous_loop = True # as long as there are commands, retrieve them diff --git a/pandaharvester/harvesterbody/event_feeder.py b/pandaharvester/harvesterbody/event_feeder.py index a46eb956..9a6319f6 100644 --- a/pandaharvester/harvesterbody/event_feeder.py +++ b/pandaharvester/harvesterbody/event_feeder.py @@ -1,4 +1,3 @@ -from future.utils import iteritems from pandaharvester.harvesterbody.agent_base import AgentBase from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils @@ -31,7 +30,7 @@ def run(self): ) mainLog.debug(f"got {len(workSpecsPerQueue)} queues") # loop over all workers - for queueName, workSpecList in iteritems(workSpecsPerQueue): + for queueName, workSpecList in workSpecsPerQueue.items(): tmpQueLog = self.make_logger(_logger, f"queue={queueName}", method_name="run") # check queue if not self.queueConfigMapper.has_queue(queueName): @@ -71,7 +70,7 @@ def run(self): tmpLog.error("failed to feed events") continue # dump - for pandaID, eventList in iteritems(events): + for pandaID, eventList in events.items(): try: nRanges = workSpec.eventsRequestParams[pandaID]["nRanges"] except Exception: diff --git a/pandaharvester/harvesterbody/job_fetcher.py b/pandaharvester/harvesterbody/job_fetcher.py index 1b1e0025..44fd2428 100644 --- a/pandaharvester/harvesterbody/job_fetcher.py +++ b/pandaharvester/harvesterbody/job_fetcher.py @@ -2,7 +2,6 @@ import random import socket -from future.utils import iteritems from pandaharvester.harvesterbody.agent_base import AgentBase from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils @@ -40,7 +39,7 @@ def run(self): pandaQueueDict = PandaQueuesDict() # loop over all queues - for queueName, nJobs in iteritems(nJobsPerQueue): + for queueName, nJobs in nJobsPerQueue.items(): # check queue if not self.queueConfigMapper.has_queue(queueName): continue @@ -78,7 +77,7 @@ def run(self): fileStatMap = dict() sw_startconvert = core_utils.get_stopwatch() for job in jobs: - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() jobSpec = JobSpec() jobSpec.convert_job_json(job) jobSpec.computingSite = queueName @@ -94,7 +93,7 @@ def run(self): if extractorCore is not None: fileGroupDictList.append(extractorCore.get_aux_inputs(jobSpec)) for fileGroupDict in fileGroupDictList: - for tmpLFN, fileAttrs in iteritems(fileGroupDict): + for tmpLFN, fileAttrs in fileGroupDict.items(): # make file spec fileSpec = FileSpec() fileSpec.PandaID = jobSpec.PandaID diff --git a/pandaharvester/harvesterbody/master.py b/pandaharvester/harvesterbody/master.py index 3794e912..7dbc8636 100644 --- a/pandaharvester/harvesterbody/master.py +++ b/pandaharvester/harvesterbody/master.py @@ -11,18 +11,18 @@ import time import daemon.pidfile -from future.utils import iteritems try: import pprofile except Exception: pass +from pandalogger import logger_config + from pandaharvester import commit_timestamp, panda_pkg_info from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestermisc.apfmon import Apfmon -from pandalogger import logger_config # logger _logger = core_utils.setup_logger("master") @@ -299,7 +299,7 @@ def main(daemon_mode=True): _logger.addHandler(stdoutHandler) # collect streams not to be closed by daemon files_preserve = [] - for loggerName, loggerObj in iteritems(logging.Logger.manager.loggerDict): + for loggerName, loggerObj in logging.Logger.manager.loggerDict.items(): if loggerName.startswith("panda"): for handler in loggerObj.handlers: if hasattr(handler, "stream"): diff --git a/pandaharvester/harvesterbody/monitor.py b/pandaharvester/harvesterbody/monitor.py index 6d6e5652..130a13e1 100644 --- a/pandaharvester/harvesterbody/monitor.py +++ b/pandaharvester/harvesterbody/monitor.py @@ -4,7 +4,6 @@ import random import time -from future.utils import iteritems from pandaharvester.harvesterbody.agent_base import AgentBase from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils @@ -98,8 +97,8 @@ def run(self): ) mainLog.debug(f"got {len(workSpecsPerQueue)} queues") # loop over all workers - for queueName, configIdWorkSpecs in iteritems(workSpecsPerQueue): - for configID, workSpecsList in iteritems(configIdWorkSpecs): + for queueName, configIdWorkSpecs in workSpecsPerQueue.items(): + for configID, workSpecsList in configIdWorkSpecs.items(): try: retVal = self.monitor_agent_core(lockedBy, queueName, workSpecsList, config_id=configID, check_source="DB") except Exception as e: @@ -162,7 +161,7 @@ def run(self): # digest events of worker update to_run_fifo_check = False retMap = self.monitor_event_digester(locked_by=lockedBy, max_events=eventBasedCheckMaxEvents) - for qc_key, retVal in iteritems(retMap): + for qc_key, retVal in retMap.items(): workSpecsToEnqueue, workSpecsToEnqueueToHead, timeNow_timestamp, fifoCheckInterval = retVal # only enqueue postprocessing workers to FIFO obj_to_enqueue_to_head_dict[qc_key][0].extend(workSpecsToEnqueueToHead) @@ -280,7 +279,7 @@ def run(self): n_chunk_put = 0 mainLog.debug("putting worker chunks to FIFO") for _dct in (obj_to_enqueue_dict, remaining_obj_to_enqueue_dict): - for (queueName, configID), obj_to_enqueue in iteritems(_dct): + for (queueName, configID), obj_to_enqueue in _dct.items(): try: workSpecsToEnqueue, timeNow_timestamp, fifoCheckInterval = obj_to_enqueue if workSpecsToEnqueue: @@ -292,7 +291,7 @@ def run(self): mainLog.error(f"failed to put object from FIFO: {errStr}") mainLog.debug("putting worker chunks to FIFO head") for _dct in (obj_to_enqueue_to_head_dict, remaining_obj_to_enqueue_to_head_dict): - for (queueName, configID), obj_to_enqueue_to_head in iteritems(_dct): + for (queueName, configID), obj_to_enqueue_to_head in _dct.items(): try: workSpecsToEnqueueToHead, timeNow_timestamp, fifoCheckInterval = obj_to_enqueue_to_head if workSpecsToEnqueueToHead: @@ -401,7 +400,7 @@ def monitor_agent_core(self, lockedBy, queueName, workSpecsList, from_fifo=False workSpec.set_work_attributes(workAttributes) workSpec.set_dialog_message(diagMessage) if isChecked: - workSpec.checkTime = datetime.datetime.utcnow() + workSpec.checkTime = core_utils.naive_utcnow() isCheckedList.append(isChecked) if monStatus == WorkSpec.ST_failed: if not workSpec.has_pilot_error() and workSpec.errorCode is None: @@ -494,7 +493,7 @@ def monitor_agent_core(self, lockedBy, queueName, workSpecsList, from_fifo=False and workSpec.mapType != WorkSpec.MT_MultiWorkers and workSpec.workAttributes is not None ): - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() timeNow_timestamp = time.time() # get lastCheckAt _bool, lastCheckAt = workSpec.get_work_params("lastCheckAt") @@ -645,7 +644,7 @@ def check_workers(self, mon_core, messenger, all_workers, queue_config, tmp_log, else: tmp_log.debug("Nothing to be checked with plugin") tmpOut = [] - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() for workSpec, (newStatus, diagMessage) in itertools.chain(zip(workersToCheck, tmpOut), thingsToPostProcess): workerID = workSpec.workerID tmp_log.debug(f"Going to check workerID={workerID}") @@ -737,7 +736,7 @@ def check_workers(self, mon_core, messenger, all_workers, queue_config, tmp_log, # retry ppTimeOut = getattr(harvester_config.monitor, "postProcessTimeout", 0) if ppTimeOut > 0: - timeLimit = datetime.datetime.utcnow() - datetime.timedelta(minutes=ppTimeOut) + timeLimit = core_utils.naive_utcnow() - datetime.timedelta(minutes=ppTimeOut) if workSpec.endTime is None or workSpec.endTime > timeLimit: isOK = False # set end time just in case for timeout @@ -794,8 +793,8 @@ def monitor_event_digester(self, locked_by, max_events): if len(workerID_list) > 0: updated_workers_dict = self.dbProxy.get_workers_from_ids(workerID_list) tmpLog.debug("got workspecs for worker events") - for queueName, _val in iteritems(updated_workers_dict): - for configID, workSpecsList in iteritems(_val): + for queueName, _val in updated_workers_dict.items(): + for configID, workSpecsList in _val.items(): qc_key = (queueName, configID) tmpLog.debug("checking workers of queueName={0} configID={1}".format(*qc_key)) try: diff --git a/pandaharvester/harvesterbody/preparator.py b/pandaharvester/harvesterbody/preparator.py index 22faea91..ae2476b5 100644 --- a/pandaharvester/harvesterbody/preparator.py +++ b/pandaharvester/harvesterbody/preparator.py @@ -129,7 +129,7 @@ def run(self): jobSpec.subStatus = "failed_to_prepare" jobSpec.lockedBy = None jobSpec.preparatorTime = None - jobSpec.stateChangeTime = datetime.datetime.utcnow() + jobSpec.stateChangeTime = core_utils.naive_utcnow() errStr = f"stage-in failed with {tmpStr}" jobSpec.set_pilot_error(PilotErrors.STAGEINFAILED, errStr) jobSpec.trigger_propagation() @@ -287,7 +287,7 @@ def run(self): jobSpec.subStatus = "failed_to_prepare" jobSpec.lockedBy = None jobSpec.preparatorTime = None - jobSpec.stateChangeTime = datetime.datetime.utcnow() + jobSpec.stateChangeTime = core_utils.naive_utcnow() errStr = f"stage-in failed with {tmpStr}" jobSpec.set_pilot_error(PilotErrors.STAGEINFAILED, errStr) jobSpec.trigger_propagation() diff --git a/pandaharvester/harvesterbody/propagator.py b/pandaharvester/harvesterbody/propagator.py index 0e6f6c9c..2cba7928 100644 --- a/pandaharvester/harvesterbody/propagator.py +++ b/pandaharvester/harvesterbody/propagator.py @@ -86,7 +86,7 @@ def run(self): # unset to disable further updating tmpJobSpec.propagatorTime = None tmpJobSpec.subStatus = "done" - tmpJobSpec.modificationTime = datetime.datetime.utcnow() + tmpJobSpec.modificationTime = core_utils.naive_utcnow() elif tmpJobSpec.is_final_status() and not tmpJobSpec.all_events_done(): # trigger next propagation to update remaining events tmpJobSpec.trigger_propagation() @@ -108,7 +108,7 @@ def run(self): tmpJobSpec.status = "cancelled" tmpJobSpec.subStatus = "killed" tmpJobSpec.set_pilot_error(PilotErrors.PANDAKILL, PilotErrors.pilot_error_msg[PilotErrors.PANDAKILL]) - tmpJobSpec.stateChangeTime = datetime.datetime.utcnow() + tmpJobSpec.stateChangeTime = core_utils.naive_utcnow() tmpJobSpec.trigger_propagation() self.dbProxy.update_job(tmpJobSpec, {"propagatorLock": self.get_pid()}, update_out_file=True) else: @@ -181,18 +181,18 @@ def run(self): else: mainLog.error(f"failed to update worker stats (bulk) for {site_name} err={tmp_str}") - if not self._last_metrics_update or datetime.datetime.utcnow() - self._last_metrics_update > datetime.timedelta(seconds=METRICS_PERIOD): + if not self._last_metrics_update or core_utils.naive_utcnow() - self._last_metrics_update > datetime.timedelta(seconds=METRICS_PERIOD): # get latest metrics from DB service_metrics_list = self.dbProxy.get_service_metrics(self._last_metrics_update) if not service_metrics_list: if self._last_metrics_update: mainLog.error("failed to get service metrics") - self._last_metrics_update = datetime.datetime.utcnow() + self._last_metrics_update = core_utils.naive_utcnow() else: tmp_ret, tmp_str = self.communicator.update_service_metrics(service_metrics_list) if tmp_ret: mainLog.debug("update of service metrics OK") - self._last_metrics_update = datetime.datetime.utcnow() + self._last_metrics_update = core_utils.naive_utcnow() else: mainLog.error(f"failed to update service metrics err={tmp_str}") diff --git a/pandaharvester/harvesterbody/service_monitor.py b/pandaharvester/harvesterbody/service_monitor.py index 78cd241b..3dbc0ed0 100644 --- a/pandaharvester/harvesterbody/service_monitor.py +++ b/pandaharvester/harvesterbody/service_monitor.py @@ -1,14 +1,10 @@ import multiprocessing import re +import subprocess import traceback import psutil -try: - import subprocess32 as subprocess -except Exception: - import subprocess - from pandaharvester.harvesterbody.agent_base import AgentBase from pandaharvester.harvesterbody.cred_manager import CredManager from pandaharvester.harvesterconfig import harvester_config diff --git a/pandaharvester/harvesterbody/submitter.py b/pandaharvester/harvesterbody/submitter.py index 37271f95..162532f1 100644 --- a/pandaharvester/harvesterbody/submitter.py +++ b/pandaharvester/harvesterbody/submitter.py @@ -3,7 +3,6 @@ import socket import time -from future.utils import iteritems from pandaharvester.harvesterbody.agent_base import AgentBase from pandaharvester.harvesterbody.worker_adjuster import WorkerAdjuster from pandaharvester.harvesterbody.worker_maker import WorkerMaker @@ -62,9 +61,9 @@ def run(self): main_log.debug(f"got {len(command_specs)} {com_str} commands") for command_spec in command_specs: new_limits = self.dbProxy.set_queue_limit(site_name, command_spec.params) - for tmp_job_type, tmp_jt_vals in iteritems(new_limits): + for tmp_job_type, tmp_jt_vals in new_limits.items(): res_map.setdefault(tmp_job_type, {}) - for tmp_resource_type, tmp_new_val in iteritems(tmp_jt_vals): + for tmp_resource_type, tmp_new_val in tmp_jt_vals.items(): # if available, overwrite new worker value with the command from panda server if tmp_resource_type in res_map[tmp_job_type]: tmp_queue_name = res_map[tmp_job_type][tmp_resource_type] @@ -210,7 +209,7 @@ def run(self): tmp_log.debug(f"successfully made {len(okChunks)} workers") else: tmp_log.debug(f"made {len(okChunks)} workers, while {len(ngChunks)} workers failed") - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() timeNow_timestamp = time.time() pandaIDs = set() # NG (=not good) @@ -412,7 +411,7 @@ def run(self): if submitted and hasattr(harvester_config.submitter, "minSubmissionInterval"): interval = harvester_config.submitter.minSubmissionInterval if interval > 0: - newTime = datetime.datetime.utcnow() + datetime.timedelta(seconds=interval) + newTime = core_utils.naive_utcnow() + datetime.timedelta(seconds=interval) self.dbProxy.update_panda_queue_attribute("submitTime", newTime, site_name=site_name) # time the cycle diff --git a/pandaharvester/harvesterbody/sweeper.py b/pandaharvester/harvesterbody/sweeper.py index 2ba6f2fc..ce519ea7 100644 --- a/pandaharvester/harvesterbody/sweeper.py +++ b/pandaharvester/harvesterbody/sweeper.py @@ -1,11 +1,6 @@ import os +from os import walk -try: - from os import walk -except ImportError: - from scandir import walk - -from future.utils import iteritems from pandaharvester.harvesterbody.agent_base import AgentBase from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils @@ -41,7 +36,7 @@ def process_kill_commands(self): for command_spec in command_specs: n_to_kill = self.dbProxy.mark_workers_to_kill_by_query(command_spec.params) tmp_log.debug(f"will kill {n_to_kill} workers with {command_spec.params}") - tmp_log.debug(f"done handling {command_string} commands took {stopwatch.get_elapsed_time()}s") + tmp_log.debug(f"done handling {command_string} commands took {stopwatch.get_elapsed_time()}") # 2. SYNC_WORKERS_KILL commands from comparing worker status provided by pilot and harvester stopwatch = core_utils.get_stopwatch() @@ -52,7 +47,7 @@ def process_kill_commands(self): for command_spec in command_specs: n_to_kill = self.dbProxy.mark_workers_to_kill_by_workerids(command_spec.params) tmp_log.debug(f"will kill {n_to_kill} workers with {command_spec.params}") - tmp_log.debug(f"done handling {command_string} commands took {stopwatch.get_elapsed_time()}s") + tmp_log.debug(f"done handling {command_string} commands took {stopwatch.get_elapsed_time()}") # main loop def run(self): @@ -75,8 +70,8 @@ def run(self): main_log.debug(f"got {len(workers_to_kill)} queues to kill workers") # loop over all workers sw = core_utils.get_stopwatch() - for queue_name, configIdWorkSpecList in iteritems(workers_to_kill): - for configID, workspec_list in iteritems(configIdWorkSpecList): + for queue_name, configIdWorkSpecList in workers_to_kill.items(): + for configID, workspec_list in configIdWorkSpecList.items(): # get sweeper if not self.queueConfigMapper.has_queue(queue_name, configID): main_log.error(f"queue config for {queue_name}/{configID} not found") @@ -140,8 +135,8 @@ def run(self): workersForCleanup = self.dbProxy.get_workers_for_cleanup(harvester_config.sweeper.maxWorkers, statusTimeoutMap) main_log.debug(f"got {len(workersForCleanup)} queues for workers cleanup") sw = core_utils.get_stopwatch() - for queue_name, configIdWorkSpecList in iteritems(workersForCleanup): - for configID, workspec_list in iteritems(configIdWorkSpecList): + for queue_name, configIdWorkSpecList in workersForCleanup.items(): + for configID, workspec_list in configIdWorkSpecList.items(): # get sweeper if not self.queueConfigMapper.has_queue(queue_name, configID): main_log.error(f"queue config for {queue_name}/{configID} not found") diff --git a/pandaharvester/harvesterbody/watcher.py b/pandaharvester/harvesterbody/watcher.py index 126be946..a72b2ae8 100644 --- a/pandaharvester/harvesterbody/watcher.py +++ b/pandaharvester/harvesterbody/watcher.py @@ -3,19 +3,15 @@ import signal import smtplib import socket +import subprocess import time - -try: - import subprocess32 as subprocess -except Exception: - import subprocess - from email.mime.text import MIMEText +from pandalogger import logger_config + from pandaharvester.harvesterbody.agent_base import AgentBase from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils -from pandalogger import logger_config logDir = logger_config.daemon["logdir"] if "PANDA_LOCK_DIR" in os.environ: @@ -32,7 +28,7 @@ class Watcher(AgentBase): # constructor def __init__(self, single_mode=False): AgentBase.__init__(self, single_mode) - self.startTime = datetime.datetime.utcnow() + self.startTime = core_utils.naive_utcnow() # main loop def run(self): @@ -46,7 +42,7 @@ def run(self): # main def execute(self): # avoid too early check - if not self.singleMode and datetime.datetime.utcnow() - self.startTime < datetime.timedelta(seconds=harvester_config.watcher.checkInterval): + if not self.singleMode and core_utils.naive_utcnow() - self.startTime < datetime.timedelta(seconds=harvester_config.watcher.checkInterval): return mainLog = core_utils.make_logger(_logger, f"id={self.get_pid()}", method_name="execute") mainLog.debug("start") @@ -64,7 +60,7 @@ def execute(self): actionsList = harvester_config.watcher.actions.split(",") for logFileName in logFileNameList: logFilePath = os.path.join(logDir, logFileName) - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() if os.path.exists(logFilePath): # get latest timestamp tmpLogDuration = None @@ -136,7 +132,7 @@ def execute(self): # message msgBody = f"harvester {harvester_config.master.harvester_id} " msgBody += f"is having a problem on {socket.getfqdn()} " - msgBody += f"at {datetime.datetime.utcnow()} (UTC)" + msgBody += f"at {core_utils.naive_utcnow()} (UTC)" message = MIMEText(msgBody) message["Subject"] = "Harvester Alarm" message["From"] = harvester_config.watcher.mailFrom diff --git a/pandaharvester/harvesterbody/worker_adjuster.py b/pandaharvester/harvesterbody/worker_adjuster.py index a533ec0f..8a006cf4 100644 --- a/pandaharvester/harvesterbody/worker_adjuster.py +++ b/pandaharvester/harvesterbody/worker_adjuster.py @@ -1,7 +1,6 @@ import copy import traceback -from future.utils import iteritems from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy @@ -83,7 +82,7 @@ def get_activate_worker_factor(self, site_name=None): # return 1/n_harvester_instances for the site val_dict = worker_stats_from_panda[site_name] n_harvester_instances = len(list(val_dict.keys())) - tmp_log.debug("number of harvesters: %s" % n_harvester_instances) + tmp_log.debug(f"number of harvesters: {n_harvester_instances}") ret_val = 1.0 / max(n_harvester_instances, 1) except KeyError: # no data for this site, return default @@ -128,8 +127,8 @@ def define_num_workers(self, static_num_workers, site_name): n_queue_total, n_ready_total, n_running_total = 0, 0, 0 apf_msg = None apf_data = None - for job_type, jt_values in iteritems(static_num_workers[queue_name]): - for resource_type, tmp_val in iteritems(jt_values): + for job_type, jt_values in static_num_workers[queue_name].items(): + for resource_type, tmp_val in jt_values.items(): tmp_log.debug(f"Processing queue {queue_name} job_type {job_type} resource_type {resource_type} with static_num_workers {tmp_val}") # set 0 to num of new workers when the queue is disabled @@ -224,8 +223,7 @@ def define_num_workers(self, static_num_workers, site_name): if job_stats[queue_name]["activated"] * self.get_activate_worker_factor(queue_name) > 0: n_min_pilots = 1 n_activated = max( - int(job_stats[queue_name]["activated"] * self.get_activate_worker_factor(queue_name)), - n_min_pilots + int(job_stats[queue_name]["activated"] * self.get_activate_worker_factor(queue_name)), n_min_pilots ) # avoid no activity queues except KeyError: # zero job in the queue diff --git a/pandaharvester/harvestercloud/pilots_starter.py b/pandaharvester/harvestercloud/pilots_starter.py index c2c585da..a8b47fa5 100644 --- a/pandaharvester/harvestercloud/pilots_starter.py +++ b/pandaharvester/harvestercloud/pilots_starter.py @@ -9,28 +9,16 @@ post-multipart code was taken from: https://github.com/haiwen/webapi-examples/blob/master/python/upload-file.py """ -try: - import subprocess32 as subprocess -except Exception: - import subprocess - -try: - import http.client as httplib # for python 3 -except Exception: - import httplib # for python 2 - -try: - import urllib.parse as urlparse # for python 3 -except ImportError: - import urlparse # for python 2 - -import os -import sys -import shutil +import http.client as httplib # for python 3 import logging import mimetypes +import os +import shutil import ssl +import subprocess +import sys import traceback +import urllib.parse as urlparse # for python 3 WORK_DIR = "/scratch" CONFIG_DIR = "/scratch/jobconfig" @@ -77,14 +65,14 @@ def encode_multipart_formdata(files): L = [] for key, filename, value in files: L.append("--" + BOUNDARY) - L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename)) - L.append("Content-Type: %s" % get_content_type(filename)) + L.append(f'Content-Disposition: form-data; name="{key}"; filename="{filename}"') + L.append(f"Content-Type: {get_content_type(filename)}") L.append("") L.append(value) L.append("--" + BOUNDARY + "--") L.append("") body = CRLF.join(L) - content_type = "multipart/form-data; boundary=%s" % BOUNDARY + content_type = f"multipart/form-data; boundary={BOUNDARY}" return content_type, body @@ -100,14 +88,14 @@ def upload_logs(url, log_file_name, destination_name, proxy_cert): logging.debug("[upload_logs] start") files = [("file", destination_name, open(log_file_name).read())] status, reason = post_multipart(urlparts.hostname, urlparts.port, urlparts.path, files, proxy_cert) - logging.debug("[upload_logs] finished with code={0} msg={1}".format(status, reason)) + logging.debug(f"[upload_logs] finished with code={status} msg={reason}") if status == 200: return True except Exception: err_type, err_value = sys.exc_info()[:2] - err_messsage = "failed to put with {0}:{1} ".format(err_type, err_value) + err_messsage = f"failed to put with {err_type}:{err_value} " err_messsage += traceback.format_exc() - logging.debug("[upload_logs] excepted with:\n {0}".format(err_messsage)) + logging.debug(f"[upload_logs] excepted with:\n {err_messsage}") return False @@ -143,45 +131,45 @@ def get_configuration(): # get the panda site name panda_site = os.environ.get("computingSite") - logging.debug("[main] got panda site: {0}".format(panda_site)) + logging.debug(f"[main] got panda site: {panda_site}") # get the panda queue name panda_queue = os.environ.get("pandaQueueName") - logging.debug("[main] got panda queue: {0}".format(panda_queue)) + logging.debug(f"[main] got panda queue: {panda_queue}") # get the resource type of the worker resource_type = os.environ.get("resourceType") - logging.debug("[main] got resource type: {0}".format(resource_type)) + logging.debug(f"[main] got resource type: {resource_type}") prodSourceLabel = os.environ.get("prodSourceLabel") - logging.debug("[main] got prodSourceLabel: {0}".format(prodSourceLabel)) + logging.debug(f"[main] got prodSourceLabel: {prodSourceLabel}") job_type = os.environ.get("jobType") - logging.debug("[main] got job type: {0}".format(job_type)) + logging.debug(f"[main] got job type: {job_type}") pilot_type = os.environ.get("pilotType", "") - logging.debug("[main] got pilotType: {0}".format(pilot_type)) + logging.debug(f"[main] got pilotType: {pilot_type}") pilot_url_option = os.environ.get("pilotUrlOpt", "") - logging.debug("[main] got pilotUrlOpt: {0}".format(pilot_url_option)) + logging.debug(f"[main] got pilotUrlOpt: {pilot_url_option}") python_option = os.environ.get("pythonOption", "") - logging.debug("[main] got pythonOption: {0}".format(python_option)) + logging.debug(f"[main] got pythonOption: {python_option}") pilot_version = os.environ.get("pilotVersion", "") - logging.debug("[main] got pilotVersion: {0}".format(pilot_version)) + logging.debug(f"[main] got pilotVersion: {pilot_version}") pilot_proxy_check_tmp = os.environ.get("pilotProxyCheck", "False") pilot_proxy_check = str_to_bool(pilot_proxy_check_tmp) - logging.debug("[main] got pilotProxyCheck: {0}".format(pilot_proxy_check)) + logging.debug(f"[main] got pilotProxyCheck: {pilot_proxy_check}") # get the Harvester ID harvester_id = os.environ.get("HARVESTER_ID") - logging.debug("[main] got Harvester ID: {0}".format(harvester_id)) + logging.debug(f"[main] got Harvester ID: {harvester_id}") # get the worker id worker_id = os.environ.get("workerID") - logging.debug("[main] got worker ID: {0}".format(worker_id)) + logging.debug(f"[main] got worker ID: {worker_id}") # get the URL (e.g. panda cache) to upload logs logs_frontend_w = os.environ.get("logs_frontend_w") @@ -194,7 +182,7 @@ def get_configuration(): # get the filename to use for the stdout log stdout_name = os.environ.get("stdout_name") if not stdout_name: - stdout_name = "{0}_{1}.out".format(harvester_id, worker_id) + stdout_name = f"{harvester_id}_{worker_id}.out" logging.debug("[main] got filename for the stdout log") @@ -253,27 +241,27 @@ def get_configuration(): ) = get_configuration() # the pilot should propagate the download link via the pilotId field in the job table - log_download_url = "{0}/{1}".format(logs_frontend_r, destination_name) + log_download_url = f"{logs_frontend_r}/{destination_name}" os.environ["GTAG"] = log_download_url # GTAG env variable is read by pilot # execute the pilot wrapper logging.debug("[main] starting pilot wrapper...") resource_type_option = "" if resource_type: - resource_type_option = "--resource-type {0}".format(resource_type) + resource_type_option = f"--resource-type {resource_type}" if prodSourceLabel: - psl_option = "-j {0}".format(prodSourceLabel) + psl_option = f"-j {prodSourceLabel}" else: psl_option = "-j managed" job_type_option = "" if job_type: - job_type_option = "--job-type {0}".format(job_type) + job_type_option = f"--job-type {job_type}" pilot_type_option = "-i PR" if pilot_type: - pilot_type_option = "-i {0}".format(pilot_type) + pilot_type_option = f"-i {pilot_type}" pilot_proxy_check_option = "-t" # This disables the proxy check if pilot_proxy_check: @@ -281,7 +269,7 @@ def get_configuration(): pilot_version_option = "--pilotversion 2" if pilot_version: - pilot_version_option = "--pilotversion {0}".format(pilot_version) + pilot_version_option = f"--pilotversion {pilot_version}" wrapper_params = "-q {0} -r {1} -s {2} -a {3} {4} {5} {6} {7} {8} {9} {10} {11}".format( panda_queue, @@ -314,7 +302,7 @@ def get_configuration(): logging.error(traceback.format_exc()) return_code = 1 - logging.debug("[main] pilot wrapper done with return code {0} ...".format(return_code)) + logging.debug(f"[main] pilot wrapper done with return code {return_code} ...") # upload logs to e.g. panda cache or similar upload_logs(logs_frontend_w, "/tmp/wrapper-wid.log", destination_name, proxy_path) diff --git a/pandaharvester/harvestercommunicator/base_communicator.py b/pandaharvester/harvestercommunicator/base_communicator.py index e6806461..87705857 100644 --- a/pandaharvester/harvestercommunicator/base_communicator.py +++ b/pandaharvester/harvestercommunicator/base_communicator.py @@ -5,7 +5,6 @@ import abc -from future.utils import with_metaclass from pandaharvester.harvestercore import core_utils # logger @@ -13,7 +12,7 @@ # base class for communication with WMS -class BaseCommunicator(with_metaclass(abc.ABCMeta, object)): +class BaseCommunicator(metaclass=abc.ABCMeta): # constructor def __init__(self): pass diff --git a/pandaharvester/harvestercommunicator/panda_communicator.py b/pandaharvester/harvestercommunicator/panda_communicator.py index 309df43b..3a4e0b4e 100644 --- a/pandaharvester/harvestercommunicator/panda_communicator.py +++ b/pandaharvester/harvestercommunicator/panda_communicator.py @@ -21,13 +21,13 @@ # TO BE REMOVED for python2.7 import requests.packages.urllib3 -from future.utils import iteritems try: requests.packages.urllib3.disable_warnings() except Exception: pass from pandacommon.pandautils.net_utils import get_http_adapter_with_random_dns_resolution + from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestermisc import idds_utils @@ -58,15 +58,15 @@ def __init__(self): def renew_token(self): if hasattr(harvester_config.pandacon, "auth_token"): if harvester_config.pandacon.auth_token.startswith("file:"): - if self.auth_token_last_update is not None and datetime.datetime.utcnow() - self.auth_token_last_update < datetime.timedelta(minutes=60): + if self.auth_token_last_update is not None and core_utils.naive_utcnow() - self.auth_token_last_update < datetime.timedelta(minutes=60): return with open(harvester_config.pandacon.auth_token.split(":")[-1]) as f: self.auth_token = f.read() - self.auth_token_last_update = datetime.datetime.utcnow() + self.auth_token_last_update = core_utils.naive_utcnow() else: if self.auth_token_last_update is None: self.auth_token = harvester_config.pandacon.auth_token - self.auth_token_last_update = datetime.datetime.utcnow() + self.auth_token_last_update = core_utils.naive_utcnow() # POST with http def post(self, path, data): @@ -197,7 +197,7 @@ def get_jobs(self, site_name, node_name, prod_source_label, computing_element, n data["nJobs"] = n_jobs data["schedulerID"] = f"harvester-{harvester_config.master.harvester_id}" if additional_criteria is not None: - for tmpKey, tmpVal in iteritems(additional_criteria): + for tmpKey, tmpVal in additional_criteria.items(): data[tmpKey] = tmpVal sw = core_utils.get_stopwatch() tmpStat, tmpRes = self.post_ssl("getJob", data) @@ -324,7 +324,7 @@ def get_event_ranges(self, data_map, scattered, base_path): getEventsChunkSize = harvester_config.pandacon.getEventsChunkSize except Exception: getEventsChunkSize = 5120 - for pandaID, data in iteritems(data_map): + for pandaID, data in data_map.items(): # get logger tmpLog = self.make_logger(f"PandaID={data['pandaID']}", method_name="get_event_ranges") if "nRanges" in data: @@ -583,7 +583,7 @@ def is_alive(self, key_values): tmpLog = self.make_logger(method_name="is_alive") tmpLog.debug("start") # convert datetime - for tmpKey, tmpVal in iteritems(key_values): + for tmpKey, tmpVal in key_values.items(): if isinstance(tmpVal, datetime.datetime): tmpVal = "datetime/" + tmpVal.strftime("%Y-%m-%d %H:%M:%S.%f") key_values[tmpKey] = tmpVal diff --git a/pandaharvester/harvesterconfig/harvester_config.py b/pandaharvester/harvesterconfig/harvester_config.py index 006de037..5e19a6c3 100644 --- a/pandaharvester/harvesterconfig/harvester_config.py +++ b/pandaharvester/harvesterconfig/harvester_config.py @@ -4,8 +4,6 @@ import socket import sys -import six -from future.utils import iteritems from liveconfigparser.LiveConfigParser import LiveConfigParser # get ConfigParser @@ -85,12 +83,12 @@ def __init__(self): # initialize config dict config_dict[tmpSection] = {} # expand all values - for tmpKey, tmpVal in iteritems(tmpDict): + for tmpKey, tmpVal in tmpDict.items(): # use env vars if isinstance(tmpVal, str) and tmpVal.startswith("$"): tmpVal = env_var_parse(tmpVal) # convert string to bool/int - if not isinstance(tmpVal, six.string_types): + if not isinstance(tmpVal, str): pass elif tmpVal == "True": tmpVal = True diff --git a/pandaharvester/harvestercore/core_utils.py b/pandaharvester/harvestercore/core_utils.py index 4fe80b9d..3cfe5b88 100644 --- a/pandaharvester/harvestercore/core_utils.py +++ b/pandaharvester/harvestercore/core_utils.py @@ -5,12 +5,12 @@ import base64 import codecs -import datetime import fcntl import functools import inspect import math import os +import pickle import random import socket import sys @@ -20,22 +20,12 @@ import uuid import zlib from contextlib import contextmanager +from datetime import datetime, timedelta, timezone +from threading import get_ident import Cryptodome.Cipher.AES import Cryptodome.Hash.HMAC import Cryptodome.Random -from future.utils import iteritems - -try: - from threading import get_ident -except ImportError: - from thread import get_ident - -try: - import cPickle as pickle -except ImportError: - import pickle - from pandalogger.LogWrapper import LogWrapper from pandalogger.PandaLogger import PandaLogger @@ -51,6 +41,10 @@ # lock for synchronization sync_lock = threading.Lock() +############## +# Decorators # +############## + # synchronize decorator def synchronize(func): @@ -64,28 +58,29 @@ def wrapper(*args, **kwargs): return wrapper +########### +# Classes # +########### + + # stopwatch class class StopWatch(object): # constructor def __init__(self): - self.startTime = datetime.datetime.utcnow() + self.start_time = time.monotonic() - # get elapsed time + # get string message about elapsed time def get_elapsed_time(self): - diff = datetime.datetime.utcnow() - self.startTime - return f" : took {diff.seconds + diff.days * 24 * 3600}.{diff.microseconds // 1000:03} sec" + time_diff = self.get_elapsed_time_in_sec() + return f" : took {time_diff:.3f} sec" # get elapsed time in seconds - def get_elapsed_time_in_sec(self, precise=False): - diff = datetime.datetime.utcnow() - self.startTime - if precise: - return diff.seconds + diff.days * 24 * 3600 + diff.microseconds * 1e-6 - else: - return diff.seconds + diff.days * 24 * 3600 + def get_elapsed_time_in_sec(self): + return time.monotonic() - self.start_time # reset def reset(self): - self.startTime = datetime.datetime.utcnow() + self.start_time = time.monotonic() # map with lock @@ -112,7 +107,11 @@ def release(self): self.lock.release() def iteritems(self): - return iteritems(self.dataMap) + return self.dataMap.items() + + +# global dict for all threads +global_dict = MapWithLock() # singleton distinguishable with id @@ -144,6 +143,26 @@ def __call__(cls, *args, **kwargs): return cls.__instance.get(obj_id) +# replacement for slow namedtuple in python 2 +class DictTupleHybrid(tuple): + def set_attributes(self, attributes): + self.attributes = attributes + + def _asdict(self): + return dict(zip(self.attributes, self)) + + +# safe dictionary to retrun original strings for missing keys +class SafeDict(dict): + def __missing__(self, key): + return "{" + key + "}" + + +############# +# Functions # +############# + + # enable memory profiling def enable_memory_profiling(): global with_memory_profile @@ -224,7 +243,7 @@ def make_pool_file_catalog(jobspec_list): doneLFNs = set() for jobSpec in jobspec_list: inFiles = jobSpec.get_input_file_attributes() - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): if inLFN in doneLFNs: continue doneLFNs.add(inLFN) @@ -356,9 +375,9 @@ def update_job_attributes_with_workers(map_type, jobspec_list, workspec_list, fi jobSpec.set_one_attribute("batchID", workSpec.batchID) # add files outFileAttrs = jobSpec.get_output_file_attributes() - for tmpWorkerID, files_to_stage_out in iteritems(files_to_stage_out_list): + for tmpWorkerID, files_to_stage_out in files_to_stage_out_list.items(): if jobSpec.PandaID in files_to_stage_out: - for lfn, fileAttersList in iteritems(files_to_stage_out[jobSpec.PandaID]): + for lfn, fileAttersList in files_to_stage_out[jobSpec.PandaID].items(): for fileAtters in fileAttersList: fileSpec = FileSpec() fileSpec.lfn = lfn @@ -446,9 +465,9 @@ def update_job_attributes_with_workers(map_type, jobspec_list, workspec_list, fi # jobSpec.set_attributes(workAttributes) # add files outFileAttrs = jobSpec.get_output_file_attributes() - for tmpWorkerID, files_to_stage_out in iteritems(files_to_stage_out_list): + for tmpWorkerID, files_to_stage_out in files_to_stage_out_list.items(): if jobSpec.PandaID in files_to_stage_out: - for lfn, fileAttersList in iteritems(files_to_stage_out[jobSpec.PandaID]): + for lfn, fileAttersList in files_to_stage_out[jobSpec.PandaID].items(): for fileAtters in fileAttersList: fileSpec = FileSpec() fileSpec.lfn = lfn @@ -508,10 +527,6 @@ def get_stopwatch(): return StopWatch() -# global dict for all threads -global_dict = MapWithLock() - - # get global dict def get_global_dict(): return global_dict @@ -531,12 +546,12 @@ def get_file_lock(file_name, lock_interval): fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) locked = True # read timestamp - timeNow = datetime.datetime.utcnow() + timeNow = naive_utcnow() toSkip = False try: s = f.read() - pTime = datetime.datetime.strptime(s, "%Y-%m-%d %H:%M:%S.%f") - if timeNow - pTime < datetime.timedelta(seconds=lock_interval): + pTime = datetime.strptime(s, "%Y-%m-%d %H:%M:%S.%f") + if timeNow - pTime < timedelta(seconds=lock_interval): toSkip = True except Exception: pass @@ -622,21 +637,12 @@ def dynamic_plugin_change(): return True -# replacement for slow namedtuple in python 2 -class DictTupleHybrid(tuple): - def set_attributes(self, attributes): - self.attributes = attributes - - def _asdict(self): - return dict(zip(self.attributes, self)) - - # Make a list of choice candidates according to permille weight def make_choice_list(pdpm={}, default=None): weight_sum = sum(pdpm.values()) weight_default = 1000 ret_list = [] - for candidate, weight in iteritems(pdpm): + for candidate, weight in pdpm.items(): if weight_sum > 1000: real_weight = int(weight * 1000 / weight_sum) else: @@ -680,7 +686,47 @@ def get_pid(): return f"{hostname}_{os_pid}-{format(get_ident(), 'x')}" -# safe dictionary to retrun original strings for missing keys -class SafeDict(dict): - def __missing__(self, key): - return "{" + key + "}" +def aware_utcnow() -> datetime: + """ + Return the current UTC date and time, with tzinfo timezone.utc + + Returns: + datetime: current UTC date and time, with tzinfo timezone.utc + """ + return datetime.now(timezone.utc) + + +def aware_utcfromtimestamp(timestamp: float) -> datetime: + """ + Return the local date and time, with tzinfo timezone.utc, corresponding to the POSIX timestamp + + Args: + timestamp (float): POSIX timestamp + + Returns: + datetime: current UTC date and time, with tzinfo timezone.utc + """ + return datetime.fromtimestamp(timestamp, timezone.utc) + + +def naive_utcnow() -> datetime: + """ + Return the current UTC date and time, without tzinfo + + Returns: + datetime: current UTC date and time, without tzinfo + """ + return aware_utcnow().replace(tzinfo=None) + + +def naive_utcfromtimestamp(timestamp: float) -> datetime: + """ + Return the local date and time, without tzinfo, corresponding to the POSIX timestamp + + Args: + timestamp (float): POSIX timestamp + + Returns: + datetime: current UTC date and time, without tzinfo + """ + return aware_utcfromtimestamp(timestamp).replace(tzinfo=None) diff --git a/pandaharvester/harvestercore/db_proxy.py b/pandaharvester/harvestercore/db_proxy.py index d4020717..342d3a5d 100644 --- a/pandaharvester/harvestercore/db_proxy.py +++ b/pandaharvester/harvestercore/db_proxy.py @@ -13,8 +13,6 @@ import threading import time -from future.utils import iteritems - from pandaharvester.harvesterconfig import harvester_config from . import core_utils @@ -742,7 +740,7 @@ def update_job(self, jobspec, criteria=None, update_in_file=False, update_out_fi sql += "WHERE PandaID=:PandaID " # update job varMap = jobspec.values_map(only_changed=True) - for tmpKey, tmpVal in iteritems(criteria): + for tmpKey, tmpVal in criteria.items(): mapKey = f":{tmpKey}_cr" sql += f"AND {tmpKey}={mapKey} " varMap[mapKey] = tmpVal @@ -851,7 +849,7 @@ def update_worker(self, workspec, criteria=None): # update worker varMap = workspec.values_map(only_changed=True) if len(varMap) > 0: - for tmpKey, tmpVal in iteritems(criteria): + for tmpKey, tmpVal in criteria.items(): mapKey = f":{tmpKey}_cr" sql += f"AND {tmpKey}={mapKey} " varMap[mapKey] = tmpVal @@ -988,7 +986,7 @@ def get_num_jobs_to_fetch(self, n_queues, interval): sqlU += "WHERE queueName=:queueName " sqlU += "AND (jobFetchTime IS NULL OR jobFetchTime<:timeLimit) " # get queues - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() varMap = dict() varMap[":timeLimit"] = timeNow - datetime.timedelta(seconds=interval) self.execute(sqlQ, varMap) @@ -1087,7 +1085,7 @@ def get_jobs_to_propagate(self, max_jobs, lock_interval, update_interval, locked sqlC = f"SELECT {FileSpec.column_names()} FROM {fileTableName} " sqlC += "WHERE PandaID=:PandaID AND fileType=:type AND status=:status " # get jobs - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() lockTimeLimit = timeNow - datetime.timedelta(seconds=lock_interval) updateTimeLimit = timeNow - datetime.timedelta(seconds=update_interval) varMap = dict() @@ -1218,7 +1216,7 @@ def get_jobs_in_sub_status( msgPfx = f"id={locked_by}" tmpLog = core_utils.make_logger(_logger, msgPfx, method_name="get_jobs_in_sub_status") tmpLog.debug(f"start subStatus={sub_status} timeColumn={time_column}") - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() # sql to count jobs being processed sqlC = f"SELECT COUNT(*) cnt FROM {jobTableName} " sqlC += f"WHERE ({lock_column} IS NOT NULL AND subStatus=:subStatus " @@ -1482,7 +1480,7 @@ def insert_workers(self, workspec_list, locked_by): tmpLog = core_utils.make_logger(_logger, f"locked_by={locked_by}", method_name="insert_workers") try: tmpLog.debug("start") - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() # sql to insert a worker sqlI = f"INSERT INTO {workTableName} ({WorkSpec.column_names()}) " sqlI += WorkSpec.bind_values_expression() @@ -1544,7 +1542,7 @@ def get_queues_to_submit(self, n_queues, lookup_interval, lock_interval, locked_ sqlU += "WHERE siteName=:siteName " sqlU += "AND (submitTime IS NULL OR submitTime<:timeLimit) " # get sites - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() varMap = dict() varMap[":lockTimeLimit"] = timeNow - datetime.timedelta(seconds=queue_lock_interval) varMap[":lookupTimeLimit"] = timeNow - datetime.timedelta(seconds=lookup_interval) @@ -1697,7 +1695,7 @@ def get_job_chunks_for_workers( sqlL = f"UPDATE {jobTableName} SET submitterTime=:timeNow,lockedBy=:lockedBy " sqlL += sqlCore sqlL += "AND PandaID=:PandaID " - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() lockTimeLimit = timeNow - datetime.timedelta(seconds=lock_interval) checkTimeLimit = timeNow - datetime.timedelta(seconds=check_interval) # sql to get file @@ -1872,7 +1870,7 @@ def get_workers_to_update(self, max_workers, check_interval, lock_interval, lock sqlP = f"SELECT PandaID FROM {jobWorkerTableName} " sqlP += "WHERE workerID=:workerID " # get workerIDs - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() lockTimeLimit = timeNow - datetime.timedelta(seconds=lock_interval) checkTimeLimit = timeNow - datetime.timedelta(seconds=check_interval) varMap = dict() @@ -2007,7 +2005,7 @@ def get_workers_to_propagate(self, max_workers, check_interval): # sql to get workers sqlG = f"SELECT {WorkSpec.column_names()} FROM {workTableName} " sqlG += "WHERE workerID=:workerID " - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() timeLimit = timeNow - datetime.timedelta(seconds=check_interval) # get workerIDs varMap = dict() @@ -2081,7 +2079,7 @@ def get_workers_to_feed_events(self, max_workers, lock_interval, locked_by): sqlG = f"SELECT {WorkSpec.column_names()} FROM {workTableName} " sqlG += "WHERE workerID=:workerID " # get workerIDs - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() lockTimeLimit = timeNow - datetime.timedelta(seconds=lock_interval) varMap = dict() varMap[":status1"] = WorkSpec.ST_running @@ -2094,7 +2092,7 @@ def get_workers_to_feed_events(self, max_workers, lock_interval, locked_by): for tmpWorkerID, tmpWorkStatus in resW: tmpWorkers[tmpWorkerID] = tmpWorkStatus retVal = {} - for workerID, workStatus in iteritems(tmpWorkers): + for workerID, workStatus in tmpWorkers.items(): # lock worker varMap = dict() varMap[":workerID"] = workerID @@ -2133,7 +2131,7 @@ def get_workers_to_feed_events(self, max_workers, lock_interval, locked_by): # update jobs and workers def update_jobs_workers(self, jobspec_list, workspec_list, locked_by, panda_ids_list=None): try: - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() # sql to check job sqlCJ = f"SELECT status FROM {jobTableName} WHERE PandaID=:PandaID FOR UPDATE " # sql to check file @@ -2530,7 +2528,7 @@ def get_jobs_with_worker_id(self, worker_id, locked_by, with_file=False, only_ru sqlF += "WHERE PandaID=:PandaID AND zipFileID IS NULL " # get jobs jobChunkList = [] - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() varMap = dict() varMap[":workerID"] = worker_id self.execute(sqlP, varMap) @@ -2727,7 +2725,7 @@ def get_jobs_for_stage_out( # sql to increment attempt number sqlFU = f"UPDATE {fileTableName} SET attemptNr=attemptNr+1 WHERE fileID=:fileID " # get jobs - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() lockTimeLimit = timeNow - datetime.timedelta(seconds=interval_with_lock) updateTimeLimit = timeNow - datetime.timedelta(seconds=interval_without_lock) varMap = dict() @@ -2867,7 +2865,7 @@ def update_job_for_stage_out(self, jobspec, update_event_status, locked_by): varMap = dict() varMap[":PandaID"] = jobspec.PandaID varMap[":lockedBy"] = locked_by - varMap[":timeNow"] = datetime.datetime.utcnow() + varMap[":timeNow"] = core_utils.naive_utcnow() self.execute(sqlLJ, varMap) nRow = self.cur.rowcount # check just in case since nRow can be 0 if two lock actions are too close in time @@ -2930,7 +2928,7 @@ def update_job_for_stage_out(self, jobspec, update_event_status, locked_by): varMap = dict() varMap[":PandaID"] = jobspec.PandaID varMap[":lockedBy"] = locked_by - varMap[":timeNow"] = datetime.datetime.utcnow() + varMap[":timeNow"] = core_utils.naive_utcnow() self.execute(sqlLJ, varMap) # commit self.commit() @@ -3114,7 +3112,7 @@ def refresh_cache(self, main_key, sub_key, new_info): tmpLog = core_utils.make_logger(_logger, f"mainKey={main_key} subKey={sub_key}", method_name="refresh_cache") # make spec cacheSpec = CacheSpec() - cacheSpec.lastUpdate = datetime.datetime.utcnow() + cacheSpec.lastUpdate = core_utils.naive_utcnow() cacheSpec.data = new_info # check if already there varMap = dict() @@ -3372,7 +3370,7 @@ def get_workers_to_kill(self, max_workers, check_interval): # sql to get workers sqlG = f"SELECT {WorkSpec.column_names()} FROM {workTableName} " sqlG += "WHERE workerID=:workerID " - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() timeLimit = timeNow - datetime.timedelta(seconds=check_interval) # get workerIDs varMap = dict() @@ -3607,10 +3605,10 @@ def mark_workers_to_kill_by_pandaid(self, panda_id, delay_seconds=None): # set time to trigger sweeper if delay_seconds is None: # set a past time to trigger sweeper immediately - setTime = datetime.datetime.utcnow() - datetime.timedelta(hours=6) + setTime = core_utils.naive_utcnow() - datetime.timedelta(hours=6) else: # set a future time to delay trigger - setTime = datetime.datetime.utcnow() + datetime.timedelta(seconds=delay_seconds) + setTime = core_utils.naive_utcnow() + datetime.timedelta(seconds=delay_seconds) # get workers varMap = dict() varMap[":pandaID"] = panda_id @@ -3651,10 +3649,10 @@ def mark_workers_to_kill_by_workerids(self, worker_ids, delay_seconds=None): # set time to trigger sweeper if delay_seconds is None: # set a past time to trigger sweeper immediately - setTime = datetime.datetime.utcnow() - datetime.timedelta(hours=6) + setTime = core_utils.naive_utcnow() - datetime.timedelta(hours=6) else: # set a future time to delay trigger - setTime = datetime.datetime.utcnow() + datetime.timedelta(seconds=delay_seconds) + setTime = core_utils.naive_utcnow() + datetime.timedelta(seconds=delay_seconds) varMaps = [] for worker_id in worker_ids: varMap = dict() @@ -3685,13 +3683,13 @@ def get_workers_for_cleanup(self, max_workers, status_timeout_map): tmpLog = core_utils.make_logger(_logger, method_name="get_workers_for_cleanup") tmpLog.debug("start") # sql to get worker IDs - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() modTimeLimit = timeNow - datetime.timedelta(minutes=60) varMap = dict() varMap[":timeLimit"] = modTimeLimit sqlW = f"SELECT workerID, configID FROM {workTableName} " sqlW += "WHERE lastUpdate IS NULL AND (" - for tmpStatus, tmpTimeout in iteritems(status_timeout_map): + for tmpStatus, tmpTimeout in status_timeout_map.items(): tmpStatusKey = f":status_{tmpStatus}" tmpTimeoutKey = f":timeLimit_{tmpStatus}" sqlW += f"(status={tmpStatusKey} AND endTime<={tmpTimeoutKey}) OR " @@ -3724,7 +3722,7 @@ def get_workers_for_cleanup(self, max_workers, status_timeout_map): sqlD = "SELECT b.lfn,b.todelete FROM {0} a, {0} b ".format(fileTableName) sqlD += "WHERE a.PandaID=:PandaID AND a.fileType IN (:fileType1,:fileType2) AND b.lfn=a.lfn " # get workerIDs - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() self.execute(sqlW, varMap) resW = self.cur.fetchall() retVal = dict() @@ -3993,9 +3991,9 @@ def set_queue_limit(self, site_name, params): ret_map = dict() queue_name = site_name - for job_type, job_values in iteritems(params): + for job_type, job_values in params.items(): ret_map.setdefault(job_type, {}) - for resource_type, value in iteritems(job_values): + for resource_type, value in job_values.items(): tmpLog.debug(f"Processing rt {resource_type} -> {value}") # get num of submitted workers @@ -4057,7 +4055,7 @@ def get_num_missed_workers(self, queue_name, criteria): sqlW += "WHERE wt.computingSite=pq.queueName AND wt.status=:status " # get worker stats varMap = dict() - for attr, val in iteritems(criteria): + for attr, val in criteria.items(): if attr == "timeLimit": sqlW += "AND wt.submitTime>:timeLimit " varMap[":timeLimit"] = val @@ -4158,7 +4156,7 @@ def get_process_lock(self, process_name, locked_by, lock_interval): sqlD = f"DELETE FROM {processLockTableName} " sqlD += "WHERE lockTime<:timeLimit " varMap = dict() - varMap[":timeLimit"] = datetime.datetime.utcnow() - datetime.timedelta(hours=6) + varMap[":timeLimit"] = core_utils.naive_utcnow() - datetime.timedelta(hours=6) self.execute(sqlD, varMap) # commit self.commit() @@ -4170,7 +4168,7 @@ def get_process_lock(self, process_name, locked_by, lock_interval): self.execute(sqlC, varMap) resC = self.cur.fetchone() retVal = False - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() if resC is None: # insert lock if missing sqlI = f"INSERT INTO {processLockTableName} ({ProcessLockSpec.column_names()}) " @@ -4468,7 +4466,7 @@ def lock_job_again(self, panda_id, time_column, lock_column, locked_by): sqlU = f"UPDATE {jobTableName} SET {time_column}=:timeNow WHERE pandaID=:pandaID " varMap = dict() varMap[":pandaID"] = panda_id - varMap[":timeNow"] = datetime.datetime.utcnow() + varMap[":timeNow"] = core_utils.naive_utcnow() self.execute(sqlU, varMap) retVal = True # commit @@ -4490,7 +4488,7 @@ def set_file_group(self, file_specs, group_id, status_string): # get logger tmpLog = core_utils.make_logger(_logger, f"groupID={group_id}", method_name="set_file_group") tmpLog.debug("start") - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() # sql to update files sqlF = f"UPDATE {fileTableName} " sqlF += "SET groupID=:groupID,groupStatus=:groupStatus,groupUpdateTime=:groupUpdateTime " @@ -4714,7 +4712,7 @@ def get_worker_ce_backend_throughput(self, site_name, time_window): sqlW += "(wt.starttime >= :timeWindowStart AND wt.starttime < :timeWindowEnd) ) " sqlW += "GROUP BY wt.status,wt.computingElement " # time window start and end - timeWindowEnd = datetime.datetime.utcnow() + timeWindowEnd = core_utils.naive_utcnow() timeWindowStart = timeWindowEnd - datetime.timedelta(seconds=time_window) timeWindowMiddle = timeWindowEnd - datetime.timedelta(seconds=time_window / 2) # get worker CE throughput @@ -4759,7 +4757,7 @@ def add_dialog_message(self, message, level, module_name, identifier=None): sqlS = f"SELECT diagID FROM {diagTableName} " sqlS += "WHERE creationTime<:timeLimit " varMap = dict() - varMap[":timeLimit"] = datetime.datetime.utcnow() - datetime.timedelta(minutes=60) + varMap[":timeLimit"] = core_utils.naive_utcnow() - datetime.timedelta(minutes=60) self.execute(sqlS, varMap) resS = self.cur.fetchall() sqlD = f"DELETE FROM {diagTableName} " @@ -4773,7 +4771,7 @@ def add_dialog_message(self, message, level, module_name, identifier=None): # make spec diagSpec = DiagSpec() diagSpec.moduleName = module_name - diagSpec.creationTime = datetime.datetime.utcnow() + diagSpec.creationTime = core_utils.naive_utcnow() diagSpec.messageLevel = level try: diagSpec.identifier = identifier[:100] @@ -4815,7 +4813,7 @@ def get_dialog_messages_to_send(self, n_messages, lock_interval): sqlM = f"SELECT {DiagSpec.column_names()} FROM {diagTableName} " sqlM += "WHERE diagID=:diagID " # select messages - timeLimit = datetime.datetime.utcnow() - datetime.timedelta(seconds=lock_interval) + timeLimit = core_utils.naive_utcnow() - datetime.timedelta(seconds=lock_interval) varMap = dict() varMap[":timeLimit"] = timeLimit self.execute(sqlD, varMap) @@ -4826,7 +4824,7 @@ def get_dialog_messages_to_send(self, n_messages, lock_interval): varMap = dict() varMap[":diagID"] = diagID varMap[":timeLimit"] = timeLimit - varMap[":timeNow"] = datetime.datetime.utcnow() + varMap[":timeNow"] = core_utils.naive_utcnow() self.execute(sqlL, varMap) nRow = self.cur.rowcount if nRow == 1: @@ -4903,8 +4901,8 @@ def delete_old_jobs(self, timeout): # get jobs varMap = dict() varMap[":subStatus"] = "done" - varMap[":timeLimit1"] = datetime.datetime.utcnow() - datetime.timedelta(hours=timeout) - varMap[":timeLimit2"] = datetime.datetime.utcnow() - datetime.timedelta(hours=timeout * 2) + varMap[":timeLimit1"] = core_utils.naive_utcnow() - datetime.timedelta(hours=timeout) + varMap[":timeLimit2"] = core_utils.naive_utcnow() - datetime.timedelta(hours=timeout * 2) self.execute(sqlGJ, varMap) resGJ = self.cur.fetchall() nDel = 0 @@ -4950,7 +4948,7 @@ def get_active_workers(self, n_workers, seconds_ago=0): sqlJ += "WHERE j.PandaID=jw.PandaID AND jw.workerID=:workerID " # parameter map varMap = dict() - varMap[":timeLimit"] = datetime.datetime.utcnow() - datetime.timedelta(seconds=seconds_ago) + varMap[":timeLimit"] = core_utils.naive_utcnow() - datetime.timedelta(seconds=seconds_ago) varMap[":st_submitted"] = WorkSpec.ST_submitted varMap[":st_running"] = WorkSpec.ST_running varMap[":st_idle"] = WorkSpec.ST_idle @@ -4988,14 +4986,14 @@ def _get_workspec_from_record(rec): # lock workers for specific thread def lock_workers(self, worker_id_list, lock_interval): try: - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() lockTimeLimit = timeNow - datetime.timedelta(seconds=lock_interval) retVal = True # get logger tmpLog = core_utils.make_logger(_logger, method_name="lock_worker") tmpLog.debug("start") # loop - for worker_id, attrs in iteritems(worker_id_list): + for worker_id, attrs in worker_id_list.items(): varMap = dict() varMap[":workerID"] = worker_id varMap[":timeNow"] = timeNow @@ -5010,7 +5008,7 @@ def lock_workers(self, worker_id_list, lock_interval): del attrs["lockedBy"] # sql to lock worker sqlL = f"UPDATE {workTableName} SET modificationTime=:timeNow" - for attrKey, attrVal in iteritems(attrs): + for attrKey, attrVal in attrs.items(): sqlL += ",{0}=:{0}".format(attrKey) varMap[f":{attrKey}"] = attrVal sqlL += " WHERE workerID=:workerID AND (lockedBy IS NULL " @@ -5041,7 +5039,7 @@ def get_queue_config_dumps(self): retVal = dict() configIDs = set() # time limit - timeLimit = datetime.datetime.utcnow() - datetime.timedelta(hours=24) + timeLimit = core_utils.naive_utcnow() - datetime.timedelta(hours=24) # get logger tmpLog = core_utils.make_logger(_logger, method_name="get_queue_config_dumps") tmpLog.debug("start") @@ -5375,7 +5373,7 @@ def lock_worker_again_to_feed_events(self, worker_id, locked_by): sqlU = f"UPDATE {workTableName} SET eventFeedTime=:timeNow WHERE workerID=:workerID " varMap = dict() varMap[":workerID"] = worker_id - varMap[":timeNow"] = datetime.datetime.utcnow() + varMap[":timeNow"] = core_utils.naive_utcnow() self.execute(sqlU, varMap) retVal = True # commit @@ -5498,7 +5496,7 @@ def get_workers_from_ids(self, ids): # sql to get associated PandaIDs sqlP = f"SELECT PandaID FROM {jobWorkerTableName} WHERE workerID=:workerID " # get workerIDs - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() varMap = dict() varMap[":st_submitted"] = WorkSpec.ST_submitted varMap[":st_running"] = WorkSpec.ST_running @@ -5595,7 +5593,7 @@ def mark_workers_to_kill_by_query(self, params, delay_seconds=None): "submissionHost": params.get("submissionHost", []), } tmpLog.debug(f"query {constraint_map}") - for attribute, match_list in iteritems(constraint_map): + for attribute, match_list in constraint_map.items(): if match_list == "ALL": pass elif not match_list: @@ -5612,10 +5610,10 @@ def mark_workers_to_kill_by_query(self, params, delay_seconds=None): # set time to trigger sweeper if delay_seconds is None: # set a past time to trigger sweeper immediately - setTime = datetime.datetime.utcnow() - datetime.timedelta(hours=6) + setTime = core_utils.naive_utcnow() - datetime.timedelta(hours=6) else: # set a future time to delay trigger - setTime = datetime.datetime.utcnow() + datetime.timedelta(seconds=delay_seconds) + setTime = core_utils.naive_utcnow() + datetime.timedelta(seconds=delay_seconds) # get workers varMap = dict() varMap.update(tmp_varMap) diff --git a/pandaharvester/harvestercore/event_spec.py b/pandaharvester/harvestercore/event_spec.py index 14fcf683..96c30ca8 100644 --- a/pandaharvester/harvestercore/event_spec.py +++ b/pandaharvester/harvestercore/event_spec.py @@ -3,9 +3,6 @@ """ -from future.utils import iteritems -from past.builtins import long - from .spec_base import SpecBase @@ -44,12 +41,12 @@ def to_data(self): # convert from data def from_data(self, data, panda_id): - for attr, val in iteritems(data): + for attr, val in data.items(): # skip non attributes if attr not in self.attributes: continue setattr(self, attr, val) - self.PandaID = long(panda_id) + self.PandaID = int(panda_id) # final status def is_final_status(self): diff --git a/pandaharvester/harvestercore/fifos.py b/pandaharvester/harvestercore/fifos.py index 0bebf44a..7d050f5d 100644 --- a/pandaharvester/harvestercore/fifos.py +++ b/pandaharvester/harvestercore/fifos.py @@ -2,21 +2,11 @@ import datetime import json import os +import pickle import socket import time from calendar import timegm - -from future.utils import iteritems - -try: - import cPickle as pickle -except ImportError: - import pickle - -try: - from threading import get_ident -except ImportError: - from thread import get_ident +from threading import get_ident from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils @@ -39,7 +29,7 @@ class FIFOBase(object): # constructor def __init__(self, **kwarg): - for tmpKey, tmpVal in iteritems(kwarg): + for tmpKey, tmpVal in kwarg.items(): setattr(self, tmpKey, tmpVal) self.hostname = socket.gethostname() self.os_pid = os.getpid() diff --git a/pandaharvester/harvestercore/job_spec.py b/pandaharvester/harvestercore/job_spec.py index 2bf0bb86..af07f7de 100644 --- a/pandaharvester/harvestercore/job_spec.py +++ b/pandaharvester/harvestercore/job_spec.py @@ -7,8 +7,7 @@ import datetime import json -from future.utils import iteritems -from past.builtins import long +from pandaharvester.harvestercore import core_utils from .spec_base import SpecBase @@ -152,15 +151,15 @@ def convert_job_json(self, data): # trigger propagation def trigger_propagation(self): - self.propagatorTime = datetime.datetime.utcnow() - datetime.timedelta(hours=1) + self.propagatorTime = core_utils.naive_utcnow() - datetime.timedelta(hours=1) # trigger preparation def trigger_preparation(self): - self.preparatorTime = datetime.datetime.utcnow() - datetime.timedelta(hours=1) + self.preparatorTime = core_utils.naive_utcnow() - datetime.timedelta(hours=1) # trigger stage out def trigger_stage_out(self): - self.stagerTime = datetime.datetime.utcnow() - datetime.timedelta(hours=1) + self.stagerTime = core_utils.naive_utcnow() - datetime.timedelta(hours=1) # set attributes def set_attributes(self, attrs): @@ -187,7 +186,7 @@ def set_attributes(self, attrs): if self.jobAttributes is None: self.jobAttributes = attrs else: - for key, val in iteritems(attrs): + for key, val in attrs.items(): if key not in self.jobAttributes or self.jobAttributes[key] != val: self.jobAttributes[key] = val self.force_update("jobAttributes") @@ -259,7 +258,7 @@ def to_event_data(self, max_events=None): data = [] eventSpecs = [] iEvents = 0 - for zipFileID, eventsData in iteritems(self.zipEventMap): + for zipFileID, eventsData in self.zipEventMap.items(): if max_events is not None and iEvents > max_events: break eventRanges = [] @@ -310,7 +309,7 @@ def get_input_file_attributes(self, skip_ready=False): endpoints = self.jobParams["ddmEndPointIn"].split(",") for lfn, guid, fsize, chksum, scope, dataset, endpoint in zip(lfns, guids, fsizes, chksums, scopes, datasets, endpoints): try: - fsize = long(fsize) + fsize = int(fsize) except Exception: fsize = None if lfn in lfnToSkip: @@ -385,16 +384,16 @@ def get_logfile_info(self): # set start time def set_start_time(self, force=False): if self.startTime is None or force is True: - self.startTime = datetime.datetime.utcnow() + self.startTime = core_utils.naive_utcnow() # set end time def set_end_time(self, force=False): if self.endTime is None or force is True: - self.endTime = datetime.datetime.utcnow() + self.endTime = core_utils.naive_utcnow() # reset start and end time def reset_start_end_time(self): - self.startTime = datetime.datetime.utcnow() + self.startTime = core_utils.naive_utcnow() self.endTime = self.startTime # add work spec list @@ -460,9 +459,9 @@ def get_job_attributes_for_panda(self): "rateWBYTES", ] panda_attributes = set(panda_attributes) - for aName, aValue in iteritems(self.jobAttributes): + for aName, aValue in self.jobAttributes.items(): if aName in panda_attributes: - if type(aValue) in (int, long): + if type(aValue) in (int,): aValue = str(aValue) data[aName] = aValue return data @@ -477,10 +476,10 @@ def get_job_status_from_attributes(self): # set group to files def set_groups_to_files(self, id_map): - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() # reverse mapping revMap = dict() - for gID, items in iteritems(id_map): + for gID, items in id_map.items(): for lfn in items["lfns"]: revMap[lfn] = gID # update file specs @@ -492,7 +491,7 @@ def set_groups_to_files(self, id_map): # update group status in files def update_group_status_in_files(self, group_id, group_status): - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() # update file specs for fileSpec in self.inFiles.union(self.outFiles): if fileSpec.groupID == group_id: @@ -563,7 +562,7 @@ def get_job_params(self, strip): return self.jobParams else: newParams = dict() - for k, v in iteritems(self.jobParams): + for k, v in self.jobParams.items(): if k in ["prodDBlocks", "realDatasetsIn", "dispatchDblock", "ddmEndPointIn", "scopeIn", "dispatchDBlockToken", "prodDBlockToken"]: continue newParams[k] = v @@ -590,14 +589,14 @@ def get_pilot_type(self): def manipulate_job_params_for_container(self): updated = False for fileSpec in self.inFiles: - for k, v in iteritems(self.jobParams): + for k, v in self.jobParams.items(): # only container image if k == "container_name": if v == fileSpec.url: self.jobParams[k] = fileSpec.path updated = True elif k == "containerOptions": - for kk, vv in iteritems(v): + for kk, vv in v.items(): if kk == "containerImage": if vv == fileSpec.url: self.jobParams[k][kk] = fileSpec.path diff --git a/pandaharvester/harvestercore/plugin_base.py b/pandaharvester/harvestercore/plugin_base.py index 286d3356..5c1d9a2e 100644 --- a/pandaharvester/harvestercore/plugin_base.py +++ b/pandaharvester/harvestercore/plugin_base.py @@ -1,10 +1,9 @@ -from future.utils import iteritems from pandaharvester.harvestercore import core_utils class PluginBase(object): def __init__(self, **kwarg): - for tmpKey, tmpVal in iteritems(kwarg): + for tmpKey, tmpVal in kwarg.items(): setattr(self, tmpKey, tmpVal) # make logger diff --git a/pandaharvester/harvestercore/plugin_factory.py b/pandaharvester/harvestercore/plugin_factory.py index 141572f8..474691b8 100644 --- a/pandaharvester/harvestercore/plugin_factory.py +++ b/pandaharvester/harvestercore/plugin_factory.py @@ -1,5 +1,3 @@ -from future.utils import iteritems - from . import core_utils from .db_interface import DBInterface @@ -45,7 +43,7 @@ def get_plugin(self, plugin_conf): self.classMap[pluginKey] = cls # make args args = {} - for tmpKey, tmpVal in iteritems(plugin_conf): + for tmpKey, tmpVal in plugin_conf.items(): if tmpKey in ["module", "name"]: continue args[tmpKey] = tmpVal diff --git a/pandaharvester/harvestercore/queue_config_mapper.py b/pandaharvester/harvestercore/queue_config_mapper.py index 90dc66e1..316dee5d 100644 --- a/pandaharvester/harvestercore/queue_config_mapper.py +++ b/pandaharvester/harvestercore/queue_config_mapper.py @@ -4,14 +4,7 @@ import json import os import threading - -import six -from future.utils import iteritems - -try: - from json.decoder import JSONDecodeError -except ImportError: - JSONDecodeError = ValueError +from json.decoder import JSONDecodeError from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestermisc.info_utils import PandaQueuesDict @@ -99,7 +92,7 @@ def set_unique_name(self): # update attributes def update_attributes(self, data): - for k, v in iteritems(data): + for k, v in data.items(): setattr(self, k, v) # get synchronization level between job and worker @@ -128,7 +121,7 @@ def __str__(self): # mapper -class QueueConfigMapper(six.with_metaclass(SingletonWithID, object)): +class QueueConfigMapper(metaclass=SingletonWithID): """ Using some acronyms here: LT = local template, written in local queueconfig file @@ -260,7 +253,7 @@ def _update_last_reload_time(self, the_time=None): got_update_lock = self.dbProxy.get_process_lock("qconf_reload", "qconf_universal", self.updateInterval) if got_update_lock: if the_time is None: - the_time = datetime.datetime.utcnow() + the_time = core_utils.naive_utcnow() ts = the_time.timestamp() new_ts_info = f"{ts:.3f}" ret_val = self.dbProxy.refresh_cache("_qconf_last_reload", "_universal", new_ts_info) @@ -277,7 +270,7 @@ def _get_last_reload_time(self): if cacheSpec is None: return None timestamp = float(cacheSpec.data) - return datetime.datetime.utcfromtimestamp(timestamp) + return core_utils.naive_utcfromtimestamp(timestamp) # update last pq_table fill time def _update_pq_table(self, cache_time=None, refill_table=False): @@ -285,7 +278,7 @@ def _update_pq_table(self, cache_time=None, refill_table=False): got_update_lock = self.dbProxy.get_process_lock("pq_table_fill", "qconf_universal", 120) if got_update_lock: fill_ret_val = self.dbProxy.fill_panda_queue_table(self.activeQueues.keys(), self, refill_table=refill_table) - now_time = datetime.datetime.utcnow() + now_time = core_utils.naive_utcnow() if fill_ret_val: now_ts = now_time.timestamp() now_ts_info = f"{now_ts:.3f}" @@ -308,7 +301,7 @@ def _get_last_pq_table_fill_time(self): if cacheSpec is None: return None timestamp = float(cacheSpec.data) - return datetime.datetime.utcfromtimestamp(timestamp) + return core_utils.naive_utcfromtimestamp(timestamp) # get time of last cache used to fill pq_table def _get_cache_to_fill_pq_table_time(self): @@ -316,14 +309,14 @@ def _get_cache_to_fill_pq_table_time(self): if cacheSpec is None: return None timestamp = float(cacheSpec.data) - return datetime.datetime.utcfromtimestamp(timestamp) + return core_utils.naive_utcfromtimestamp(timestamp) # load data def load_data(self, refill_table=False): mainLog = _make_logger(token=f"id={core_utils.get_pid()}", method_name="load_data") # check if to update with self.lock: - now_time = datetime.datetime.utcnow() + now_time = core_utils.naive_utcnow() updateInterval_td = datetime.timedelta(seconds=self.updateInterval) checkInterval_td = datetime.timedelta(seconds=self.checkInterval) # skip if lastCheck is fresh (within checkInterval) @@ -346,7 +339,7 @@ def load_data(self, refill_table=False): # start with self.lock: # update timestamp of last reload, lock with check interval - now_time = datetime.datetime.utcnow() + now_time = core_utils.naive_utcnow() update_last_reload_ret_val = self._update_last_reload_time(now_time) if update_last_reload_ret_val: self.lastReload = now_time @@ -378,7 +371,7 @@ def load_data(self, refill_table=False): mainLog.warning(f"Found cacher data outdated ({str(self.last_cache_ts)} < {str(self.cache_to_fill_pq_table_time)})") if queueConfigJson_cacher is not None: mainLog.debug("Applying cacher data") - for queueName, queueDict in iteritems(queueConfigJson_cacher): + for queueName, queueDict in queueConfigJson_cacher.items(): if queueDict.get("isTemplateQueue") is True or queueName.endswith("_TEMPLATE"): # is RT queueDict["isTemplateQueue"] = True @@ -392,7 +385,7 @@ def load_data(self, refill_table=False): queueConfigJson_local = self._load_config_from_file() if queueConfigJson_local is not None: mainLog.debug("Applying local config") - for queueName, queueDict in iteritems(queueConfigJson_local): + for queueName, queueDict in queueConfigJson_local.items(): if queueDict.get("isTemplateQueue") is True or queueName.endswith("_TEMPLATE"): # is LT queueDict["isTemplateQueue"] = True @@ -410,7 +403,7 @@ def load_data(self, refill_table=False): finalTemplatesDict.pop(None, None) # remove queues with invalid templateQueueName for acr, queuesDict in [("RQ", remoteQueuesDict), ("LQ", localQueuesDict)]: - for queueName, queueDict in iteritems(queuesDict.copy()): + for queueName, queueDict in queuesDict.copy().items(): templateQueueName = queueDict.get("templateQueueName") if templateQueueName is not None and templateQueueName not in finalTemplatesDict: del queuesDict[queueName] @@ -440,7 +433,7 @@ def load_data(self, refill_table=False): continue # parameters resolver_harvester_params = resolver.get_harvester_params(queueName) - for key, val in iteritems(resolver_harvester_params): + for key, val in resolver_harvester_params.items(): if key in self.dynamic_queue_generic_attrs: queueDict[key] = val # fill in dynamic queue configs @@ -479,7 +472,7 @@ def load_data(self, refill_table=False): continue queueSourceList.append(acr) tmp_queueDict = queuesDict[queueName] - for key, val in iteritems(tmp_queueDict): + for key, val in tmp_queueDict.items(): val = copy.deepcopy(val) if key in self.updatable_plugin_attrs and isinstance(queueDict.get(key), dict) and isinstance(val, dict): # update plugin parameters instead of overwriting whole plugin section @@ -507,12 +500,12 @@ def load_data(self, refill_table=False): if isinstance(queueDict.get("common"), dict): commonAttrDict = queueDict.get("common") # according to queueDict - for key, val in iteritems(queueDict): + for key, val in queueDict.items(): if isinstance(val, dict) and "module" in val and "name" in val: # plugin attributes val = copy.deepcopy(val) # fill in common attributes for all plugins - for c_key, c_val in iteritems(commonAttrDict): + for c_key, c_val in commonAttrDict.items(): if c_key not in val and c_key not in ("module", "name"): val[c_key] = c_val # check module and class name @@ -536,7 +529,7 @@ def load_data(self, refill_table=False): # keep original config val["original_config"] = copy.deepcopy(val) # overwrite with middleware config - for m_key, m_val in iteritems(queueDict[val["middleware"]]): + for m_key, m_val in queueDict[val["middleware"]].items(): val[m_key] = m_val setattr(queueConfig, key, val) # delete isTemplateQueue attribute @@ -595,7 +588,7 @@ def load_data(self, refill_table=False): queueConfigDumps = self.dbProxy.get_queue_config_dumps() # get active queues activeQueues = dict() - for queueName, queueConfig in iteritems(newQueueConfig): + for queueName, queueConfig in newQueueConfig.items(): # get status if queueConfig.queueStatus is None and autoBlacklist: queueConfig.queueStatus = resolver.get_queue_status(queueName) @@ -617,7 +610,7 @@ def load_data(self, refill_table=False): dumpSpec = queueConfigDumps[dumpSpec.dumpUniqueName] else: # add dump - dumpSpec.creationTime = datetime.datetime.utcnow() + dumpSpec.creationTime = core_utils.naive_utcnow() dumpSpec.configID = self.dbProxy.get_next_seq_number("SEQ_configID") tmpStat = self.dbProxy.add_queue_config_dump(dumpSpec) if not tmpStat: @@ -659,7 +652,7 @@ def load_data(self, refill_table=False): newQueueConfigWithID[dumpSpec.configID] = queueConfig self.queueConfigWithID = newQueueConfigWithID # update lastUpdate and lastCheck - self.lastUpdate = datetime.datetime.utcnow() + self.lastUpdate = core_utils.naive_utcnow() self.lastCheck = self.lastUpdate # update database pq_table if self.toUpdateDB: @@ -709,7 +702,7 @@ def get_active_ups_queues(self): """ active_ups_queues = [] active_queues = self.get_active_queues() - for queue_name, queue_attribs in iteritems(active_queues): + for queue_name, queue_attribs in active_queues.items(): try: if queue_attribs.runMode == "slave" and queue_attribs.mapType == "NoJob": active_ups_queues.append(queue_name) diff --git a/pandaharvester/harvestercore/resource_type_mapper.py b/pandaharvester/harvestercore/resource_type_mapper.py index 54b5f2bb..5d3799ee 100644 --- a/pandaharvester/harvestercore/resource_type_mapper.py +++ b/pandaharvester/harvestercore/resource_type_mapper.py @@ -1,9 +1,9 @@ -from __future__ import division - import datetime import math import threading +from pandaharvester.harvestercore import core_utils + from .db_proxy_pool import DBProxyPool as DBProxy @@ -31,7 +31,7 @@ def __init__(self): def load_data(self): with self.lock: # check interval - time_now = datetime.datetime.utcnow() + time_now = core_utils.naive_utcnow() if self.last_update is not None and time_now - self.last_update < datetime.timedelta(minutes=10): return @@ -50,7 +50,7 @@ def load_data(self): except KeyError: continue - self.last_update = datetime.datetime.utcnow() + self.last_update = core_utils.naive_utcnow() return def is_valid_resource_type(self, resource_name): diff --git a/pandaharvester/harvestercore/service_metrics_spec.py b/pandaharvester/harvestercore/service_metrics_spec.py index bb1738f2..03872f61 100644 --- a/pandaharvester/harvestercore/service_metrics_spec.py +++ b/pandaharvester/harvestercore/service_metrics_spec.py @@ -6,6 +6,8 @@ import json import socket +from pandaharvester.harvestercore import core_utils + from .spec_base import SpecBase @@ -21,6 +23,6 @@ class ServiceMetricSpec(SpecBase): def __init__(self, service_metrics): SpecBase.__init__(self) - self.creationTime = datetime.datetime.utcnow() + self.creationTime = core_utils.naive_utcnow() self.hostName = socket.getfqdn() self.metrics = service_metrics # blobs are automatically translated to json diff --git a/pandaharvester/harvestercore/spec_base.py b/pandaharvester/harvestercore/spec_base.py index d3961ca5..dcda9337 100644 --- a/pandaharvester/harvestercore/spec_base.py +++ b/pandaharvester/harvestercore/spec_base.py @@ -5,14 +5,9 @@ import json import pickle +from json.decoder import JSONDecodeError import rpyc -from future.utils import iteritems - -try: - from json.decoder import JSONDecodeError -except ImportError: - JSONDecodeError = ValueError # encoder for non-native json objects @@ -207,5 +202,5 @@ def get_changed_attributes(self): # set attributes def set_attributes_with_dict(self, attr_dict): - for attr, val in iteritems(attr_dict): + for attr, val in attr_dict.items(): setattr(self, attr, val) diff --git a/pandaharvester/harvestercore/work_spec.py b/pandaharvester/harvestercore/work_spec.py index dbc7ed0f..ad89efd9 100644 --- a/pandaharvester/harvestercore/work_spec.py +++ b/pandaharvester/harvestercore/work_spec.py @@ -7,8 +7,8 @@ import os import re -from future.utils import iteritems from pandaharvester.harvesterconfig import harvester_config +from pandaharvester.harvestercore import core_utils from .spec_base import SpecBase @@ -203,11 +203,11 @@ def post_processed(self): # trigger next lookup def trigger_next_lookup(self): self.nextLookup = True - self.modificationTime = datetime.datetime.utcnow() - datetime.timedelta(hours=1) + self.modificationTime = core_utils.naive_utcnow() - datetime.timedelta(hours=1) # trigger propagation def trigger_propagation(self): - self.lastUpdate = datetime.datetime.utcnow() - datetime.timedelta(hours=24) + self.lastUpdate = core_utils.naive_utcnow() - datetime.timedelta(hours=24) # disable propagation def disable_propagation(self): @@ -262,12 +262,12 @@ def convert_to_propagate(self): # set start time def set_start_time(self, force=False): if self.startTime is None or force is True: - self.startTime = datetime.datetime.utcnow() + self.startTime = core_utils.naive_utcnow() # set end time def set_end_time(self, force=False): if self.endTime is None or force is True: - self.endTime = datetime.datetime.utcnow() + self.endTime = core_utils.naive_utcnow() # set work params def set_work_params(self, data): @@ -275,7 +275,7 @@ def set_work_params(self, data): return if self.workParams is None and data is not None: self.workParams = dict() - for key, val in iteritems(data): + for key, val in data.items(): if key not in self.workParams or self.workParams[key] != val: self.workParams[key] = val self.force_update("workParams") @@ -298,7 +298,7 @@ def set_work_attributes(self, data): return if self.workAttributes is None and data is not None: self.workAttributes = dict() - for key, val in iteritems(data): + for key, val in data.items(): if key not in self.workAttributes or self.workAttributes[key] != val: self.workAttributes[key] = val self.force_update("workAttributes") @@ -321,7 +321,7 @@ def update_log_files_to_upload(self, file_path, position, remote_name=None, stre self.logFilesToUpload = dict() if stream_type is not None: # delete existing stream - for tmp_file_path, tmpDict in iteritems(self.logFilesToUpload.copy()): + for tmp_file_path, tmpDict in self.logFilesToUpload.copy().items(): if tmpDict["stream_type"] == stream_type: del self.logFilesToUpload[tmp_file_path] if file_path not in self.logFilesToUpload: @@ -354,7 +354,7 @@ def set_log_file(self, log_type, stream): def get_log_files_to_upload(self): retList = [] if self.logFilesToUpload is not None: - for filePath, fileInfo in iteritems(self.logFilesToUpload): + for filePath, fileInfo in self.logFilesToUpload.items(): if not os.path.exists(filePath): continue fileSize = os.stat(filePath).st_size diff --git a/pandaharvester/harvestercredmanager/grid_cred_manager.py b/pandaharvester/harvestercredmanager/grid_cred_manager.py index 26d509a1..113a7dfc 100644 --- a/pandaharvester/harvestercredmanager/grid_cred_manager.py +++ b/pandaharvester/harvestercredmanager/grid_cred_manager.py @@ -1,7 +1,4 @@ -try: - import subprocess32 as subprocess -except Exception: - import subprocess +import subprocess from pandaharvester.harvestercore import core_utils diff --git a/pandaharvester/harvestercredmanager/lancium_cred_manager.py b/pandaharvester/harvestercredmanager/lancium_cred_manager.py deleted file mode 100644 index 34f4c53b..00000000 --- a/pandaharvester/harvestercredmanager/lancium_cred_manager.py +++ /dev/null @@ -1,88 +0,0 @@ -import json -import os -import socket -import traceback - -from pandaharvester.harvestercore import core_utils -from pandaharvester.harvestermisc.info_utils import PandaQueuesDict -from pandaharvester.harvestermisc.lancium_utils import SECRETS_PATH, LanciumClient - -from .base_cred_manager import BaseCredManager - -# logger -_logger = core_utils.setup_logger("lancium_cred_manager") - - -# upload cred to Lancium periodically -class LanciumCredManager(BaseCredManager): - def __init__(self, **kwarg): - self.hostname = socket.getfqdn() - BaseCredManager.__init__(self, **kwarg) - - tmp_log = self.make_logger(_logger, method_name="__init__") - - # attributes - if hasattr(self, "inFile") or hasattr(self, "inCertFile"): - # set up with json in inFile - try: - self.inFile - except AttributeError: - self.inFile = self.inCertFile - # parse inFile setup configuration - try: - with open(self.inFile) as f: - self.setupMap = json.load(f) - except Exception as e: - tmp_log.error(f"Error with inFile/inCertFile . {e.__class__.__name__}: {e}") - self.setupMap = {} - raise - else: - # set up with direct attributes - self.setupMap = dict(vars(self)) - # validate setupMap - try: - self.proxy_files = self.setupMap["proxy_files"] - self.secret_name = self.setupMap.get("secret_name", "proxy-secret") - except KeyError as e: - tmp_log.error(f"Missing attributes in setup. {e.__class__.__name__}: {e}") - raise - - try: - self.panda_queues_dict = PandaQueuesDict() - self.lancium_client = LanciumClient(self.hostname, queue_name=self.queueName) - except Exception as e: - tmp_log.error("Problem instantiating lancium client. {1}".format(traceback.format_exc())) - raise - - # check proxy - def check_credential(self): - # same update period as credmanager agent - return False - - def upload_proxies(self, proxy_files): - tmp_log = self.make_logger(_logger, method_name="upload_proxies") - - tmp_log.debug("Start uploading proxies") - for local_file in proxy_files: - try: - tmp_log.debug(f"Uploading proxy {local_file}...") - base_name = os.path.basename(local_file) - lancium_file = os.path.join(SECRETS_PATH, base_name) - self.lancium_client.upload_file(local_file, lancium_file) - except Exception: - tmp_log.error(f"Problem uploading proxy {local_file}. {traceback.format_exc()}") - - tmp_log.debug("Done uploading proxies") - - # renew proxy - def renew_credential(self): - tmp_log = self.make_logger(_logger, f"queueName={self.queueName}", method_name="renew_credential") - - try: - self.upload_proxies(self.proxy_files) - tmp_log.debug("done") - except KeyError as e: - err_str = f"Error renewing proxy secret. {e.__class__.__name__}: {e}" - return False, err_str - else: - return True, "" diff --git a/pandaharvester/harvestercredmanager/no_voms_cred_manager.py b/pandaharvester/harvestercredmanager/no_voms_cred_manager.py index d3724630..b441c382 100644 --- a/pandaharvester/harvestercredmanager/no_voms_cred_manager.py +++ b/pandaharvester/harvestercredmanager/no_voms_cred_manager.py @@ -1,7 +1,4 @@ -try: - import subprocess32 as subprocess -except Exception: - import subprocess +import subprocess from pandaharvester.harvestercore import core_utils diff --git a/pandaharvester/harvestercredmanager/proxy_cache_cred_manager.py b/pandaharvester/harvestercredmanager/proxy_cache_cred_manager.py index e761980d..23e6fe7d 100644 --- a/pandaharvester/harvestercredmanager/proxy_cache_cred_manager.py +++ b/pandaharvester/harvestercredmanager/proxy_cache_cred_manager.py @@ -1,7 +1,4 @@ -try: - import subprocess32 as subprocess -except Exception: - import subprocess +import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.communicator_pool import CommunicatorPool diff --git a/pandaharvester/harvesterfifo/sqlite_fifo.py b/pandaharvester/harvesterfifo/sqlite_fifo.py index 73068fb9..5fd4b1d1 100644 --- a/pandaharvester/harvesterfifo/sqlite_fifo.py +++ b/pandaharvester/harvesterfifo/sqlite_fifo.py @@ -2,11 +2,7 @@ import re import sqlite3 import time - -try: - from threading import get_ident -except ImportError: - from thread import get_ident +from threading import get_ident from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore.plugin_base import PluginBase diff --git a/pandaharvester/harvestermessenger/act_messenger.py b/pandaharvester/harvestermessenger/act_messenger.py index 7b085caa..34dd6531 100644 --- a/pandaharvester/harvestermessenger/act_messenger.py +++ b/pandaharvester/harvestermessenger/act_messenger.py @@ -3,6 +3,7 @@ import os from act.atlas.aCTDBPanda import aCTDBPanda + from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.work_spec import WorkSpec @@ -118,7 +119,7 @@ def events_to_update(self, workspec): newDict = dict() # change the key from str to int for tmpPandaID, tmpDict in tmpOrigDict.items(): - tmpPandaID = long(tmpPandaID) + tmpPandaID = int(tmpPandaID) retDict[tmpPandaID] = tmpDict nData += len(tmpDict) except Exception as x: @@ -143,14 +144,14 @@ def acknowledge_events_files(self, workspec): try: jsonFilePath = os.path.join(accessPoint, jsonEventsUpdateFileName) jsonFilePath += suffixReadJson - jsonFilePath_rename = jsonFilePath + "." + str(datetime.datetime.utcnow()) + jsonFilePath_rename = jsonFilePath + "." + str(core_utils.naive_utcnow()) os.rename(jsonFilePath, jsonFilePath_rename) except Exception: pass try: jsonFilePath = os.path.join(accessPoint, jsonOutputsFileName) jsonFilePath += suffixReadJson - jsonFilePath_rename = jsonFilePath + "." + str(datetime.datetime.utcnow()) + jsonFilePath_rename = jsonFilePath + "." + str(core_utils.naive_utcnow()) os.rename(jsonFilePath, jsonFilePath_rename) except Exception: pass diff --git a/pandaharvester/harvestermessenger/arc_messenger.py b/pandaharvester/harvestermessenger/arc_messenger.py deleted file mode 100644 index d8f26f93..00000000 --- a/pandaharvester/harvestermessenger/arc_messenger.py +++ /dev/null @@ -1,550 +0,0 @@ -import errno -import os -import json -import re -import shutil -import tarfile -from urlparse import urlparse -import arc - -from pandaharvester.harvestercore import core_utils -from .base_messenger import BaseMessenger -from pandaharvester.harvesterconfig import harvester_config -from pandaharvester.harvestermisc import arc_utils -from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper -from pandaharvester.harvestercore.work_spec import WorkSpec - - -# json for outputs -jsonOutputsFileName = harvester_config.payload_interaction.eventStatusDumpJsonFile - -# xml for outputs -xmlOutputsBaseFileName = harvester_config.payload_interaction.eventStatusDumpXmlFile - -# json for event request -jsonEventsRequestFileName = harvester_config.payload_interaction.eventRequestFile - -# json to feed events -jsonEventsFeedFileName = harvester_config.payload_interaction.eventRangesFile - -# json to update events -jsonEventsUpdateFileName = harvester_config.payload_interaction.updateEventsFile - -# suffix to read json -suffixReadJson = '.read' - -# logger -baselogger = core_utils.setup_logger() - -class ARCMessenger(BaseMessenger): - '''Mechanism for passing information about completed jobs back to harvester.''' - - def __init__(self, **kwarg): - self.jobSpecFileFormat = 'json' - BaseMessenger.__init__(self, **kwarg) - self.schedulerid = harvester_config.master.harvester_id - self.tmpdir = '/tmp' # TODO configurable or common function - - # Credential dictionary role: proxy file - self.certs = dict(zip([r.split('=')[1] for r in list(harvester_config.credmanager.voms)], - list(harvester_config.credmanager.outCertFile))) - self.cred_type = arc.initializeCredentialsType(arc.initializeCredentialsType.SkipCredentials) - - - def _setup_proxy(self, usercfg, workspec, jobid, log): - '''Set up UserConfig object with correct proxy''' - - proxyrole = workspec.workAttributes['proxyrole'] - try: - usercfg.ProxyPath(str(self.certs[proxyrole])) - except: - log.error("Job {0}: no proxy found with role {1}".format(jobid, proxyrole)) - return False - return True - - - def _copy_file(self, source, destination, usercfg, log): - '''Copy a file from source to destination''' - - log.info('Copying {0} to {1}'.format(source, destination)) - source_datapoint = arc_utils.DataPoint(str(source), usercfg) - destination_datapoint = arc_utils.DataPoint(str(destination), usercfg) - dm = arc.DataMover() - dm.retry(False) - dm.passive(True) - dm.secure(False) - - status = dm.Transfer(source_datapoint.h, destination_datapoint.h, arc.FileCache(), arc.URLMap()) - return status - - - def _delete_file(self, filename, usercfg, log): - '''Delete a remote file on ARC CE''' - log.info('Deleting {0}'.format(filename)) - datapoint = arc_utils.DataPoint(str(filename), usercfg) - datapoint.h.SetSecure(False) - status = datapoint.h.Remove() - return status - - - def _list_url_recursive(self, url, log, fname='', filelist=[]): - '''List ARC job directory recursively to find all files''' - - dp = arc_utils.DataPoint(url+'/'+fname, self.userconfig) - files = dp.h.List(arc.DataPoint.INFO_TYPE_NAME | arc.DataPoint.INFO_TYPE_TYPE) - if not files[1]: - log.warning("Failed listing %s/%s" % (url, fname)) - return filelist - for f in files[0]: - if f.GetType() == f.file_type_file: - filelist.append((fname+'/'+f.GetName()).strip('/')) - elif f.GetType() == f.file_type_dir: - filelist = self.listUrlRecursive(url, log, (fname+'/'+str(f.GetName())).strip('/'), filelist) - return filelist - - - def _download_outputs(self, files, logdir, jobid, pandaid, userconfig, log): - '''Download the output files specified in downloadfiles''' - - # construct datapoint object, initialising connection. Use the same - # object until base URL changes. TODO group by base URL. - - datapoint = arc_utils.DataPoint(str(jobid), userconfig) - dp = datapoint.h - dm = arc.DataMover() - dm.retry(False) - dm.passive(True) - dm.secure(False) - fetched = [] - notfetched = [] - notfetchedretry = [] - - # create required local log dirs - try: - os.makedirs(logdir, 0755) - except OSError as e: - if e.errno != errno.EEXIST or not os.path.isdir(logdir): - log.warning('Failed to create directory {0}: {1}'.format(logdir, os.strerror(e.errno))) - notfetched.append(jobid) - return (fetched, notfetched, notfetchedretry) - - tmpdldir = os.path.join(self.tmpdir, pandaid) - try: - os.makedirs(tmpdldir, 0755) - except OSError as e: - if e.errno != errno.EEXIST or not os.path.isdir(tmpdldir): - log.warning('Failed to create directory {0}: {1}'.format(tmpdldir, os.strerror(e.errno))) - notfetched.append(jobid) - return (fetched, notfetched, notfetchedretry) - - filelist = files.split(';') - if re.search(r'[\*\[\]\?]', files): - # found wildcard, need to get sessiondir list - remotefiles = self.listUrlRecursive(jobid, log) - expandedfiles = [] - for wcf in filelist: - if re.search(r'[\*\[\]\?]', wcf): - # only match wildcards in matching dirs - expandedfiles += [rf for rf in remotefiles if fnmatch.fnmatch(rf, wcf) and os.path.dirname(rf) == os.path.dirname(wcf)] - else: - expandedfiles.append(wcf) - # remove duplicates from wildcard matching through set - filelist = list(set(expandedfiles)) - - for f in filelist: - if f == 'gmlog/errors': - localfile = os.path.join(logdir, '%s.log' % pandaid) - elif f.find('.log') != -1: - localfile = os.path.join(logdir, '%s.out' % pandaid) - else: - localfile = os.path.join(tmpdldir, f) - - remotefile = arc.URL(str(jobid + '/' + f)) - dp.SetURL(remotefile) - localdp = arc_utils.DataPoint(str(localfile), userconfig) - # do the copy - status = dm.Transfer(dp, localdp.h, arc.FileCache(), arc.URLMap()) - if not status and str(status).find('File unavailable') == -1: # tmp fix for globus error which is always retried - if status.Retryable(): - log.warning('Failed to download but will retry {0}: {1}'.format(dp.GetURL().str(), str(status))) - notfetchedretry.append(jobid) - else: - log.error('Failed to download with permanent failure {0}: {1}'.format(dp.GetURL().str(), str(status))) - notfetched.append(jobid) - else: - os.chmod(localfile, 0644) - log.info('Downloaded {0}'.format(dp.GetURL().str())) - - if jobid not in notfetched and jobid not in notfetchedretry: - fetched.append(jobid) - - return (fetched, notfetched, notfetchedretry) - - - def _extractAndFixPilotPickle(self, arcjob, pandaid, haveoutput, logurl, log): - ''' - Extract the pilot pickle from jobSmallFiles.tgz, and fix attributes - ''' - - arcid = arcjob['JobID'] - pandapickle = None - tmpjobdir = os.path.join(self.tmpdir, pandaid) - if haveoutput: - log.debug('os.cwd(): {0}'.format(os.getcwd())) - try: - smallfiles = tarfile.open(os.path.join(tmpjobdir, 'jobSmallFiles.tgz')) - pandapickle = smallfiles.extractfile("panda_node_struct.pickle") - except Exception as e: - log.error("{0}: failed to extract pickle for arcjob {1}: {2}".format(pandaid, arcid, str(e))) - - if pandapickle: - jobinfo = arc_utils.ARCPandaJob(filehandle=pandapickle) - # de-serialise the metadata to json - try: - jobinfo.metaData = json.loads(jobinfo.metaData) - except: - log.warning("{0}: no metaData in pilot pickle".format(pandaid)) - shutil.rmtree(tmpjobdir, ignore_errors=True) - else: - jobinfo = arc_utils.ARCPandaJob(jobinfo={'jobId': long(pandaid), 'state': 'finished'}) - jobinfo.schedulerID = self.schedulerid - if logurl: - jobinfo.pilotID = "%s.out|Unknown|Unknown|Unknown|Unknown" % '/'.join([logurl, pandaid]) - - # TODO: set error code based on batch error message (memory kill etc) - jobinfo.pilotErrorCode = 1008 - if arcjob['Error']: - jobinfo.pilotErrorDiag = arcjob['Error'] - else: - # Probably failure getting outputs - jobinfo.pilotErrorDiag = "Failed to get outputs from CE" - - jobinfo.computingElement = urlparse(arcid).netloc - - return jobinfo.dictionary() - - - def get_access_point(self, workspec, panda_id): - '''Get access point''' - if workspec.mapType == WorkSpec.MT_MultiJobs: - accessPoint = os.path.join(workspec.get_access_point(), str(panda_id)) - else: - accessPoint = workspec.get_access_point() - return accessPoint - - - def post_processing(self, workspec, jobspec_list, map_type): - ''' - Fetch job output and process pilot info for sending in final heartbeat. - The pilot pickle is loaded and some attributes corrected (schedulerid, - pilotlog etc), then converted to dictionary and stored in - workspec.workAttributes[pandaid]. If pilot pickle cannot be used, - report ARC error in pilotErrorDiag and fill all possible attributes - using ARC information. - ''' - - arclog = arc_utils.ARCLogger(baselogger, workspec.workerID) - tmplog = arclog.log - tmplog.info('Post processing ARC job {0}'.format(workspec.batchID)) - job = workspec.workAttributes['arcjob'] - arcid = job['JobID'] - tmplog.info('Job id {0}'.format(arcid)) - - if 'arcdownloadfiles' not in workspec.workAttributes: - tmplog.error('No files to download') - return True - - # Assume one-to-one mapping of workers to jobs. If jobspec_list is empty - # it means the job was cancelled by panda or otherwise forgotten - if not jobspec_list: - return True - - # Set certificate to use for interacting with ARC CE - userconfig = arc.UserConfig(self.cred_type) - if not self._setup_proxy(usercfg, workspec, arcid, tmplog): - return True - - queueconfigmapper = QueueConfigMapper() - queueconfig = queueconfigmapper.get_queue(jobspec_list[0].computingSite) - logbaseurl = queueconfig.submitter.get('logBaseURL') - logbasedir = queueconfig.submitter.get('logDir', self.tmpdir) - logsubdir = workspec.workAttributes['logsubdir'] - pandaid = str(jobspec_list[0].PandaID) - - # Construct log path and url - logurl = '/'.join([logbaseurl, logsubdir, str(pandaid)]) if logbaseurl else None - logdir = os.path.join(logbasedir, logsubdir) - - # post_processing is only called once, so no retries are done. But keep - # the possibility here in case it changes - (fetched, notfetched, notfetchedretry) = self._download_outputs(workspec.workAttributes['arcdownloadfiles'], - logdir, arcid, pandaid, userconfig, tmplog) - if arcid not in fetched: - tmplog.warning("Could not get outputs of {0}".format(arcid)) - - workspec.workAttributes[long(pandaid)] = {} - - workspec.workAttributes[long(pandaid)] = self._extractAndFixPilotPickle(job, pandaid, (arcid in fetched), logurl, tmplog) - - tmplog.debug("pilot info for {0}: {1}".format(pandaid, workspec.workAttributes[long(pandaid)])) - return True - - - def get_work_attributes(self, workspec): - '''Get info from the job to pass back to harvester''' - # Just return existing attributes. Attributes are added to workspec for - # finished jobs in post_processing - return workspec.workAttributes - - - def events_requested(self, workspec): - '''Used to tell harvester that the worker requests events''' - - # get logger - arclog = arc_utils.ARCLogger(baselogger, workspec.workerID) - tmpLog = arclog.log - - # Check for jobid/jsonEventsRequestFileName - job = workspec.workAttributes['arcjob'] - arcid = job['JobID'] - # Set certificate to use for interacting with ARC CE - usercfg = arc.UserConfig(self.cred_type) - if not self._setup_proxy(usercfg, workspec, arcid, tmpLog): - return {} - - remoteJsonFilePath = '%s/%s' % (arcid, jsonEventsRequestFileName) - localJsonFilePath = os.path.join(workspec.get_access_point(), jsonEventsRequestFileName) - tmpLog.debug('looking for event request file {0}'.format(remoteJsonFilePath)) - # Try to copy the file - status = self._copy_file(remoteJsonFilePath, localJsonFilePath, usercfg, tmpLog) - if not status: - if status.GetErrno() == errno.ENOENT: - # Not found - tmpLog.debug('not found') - return {} - # Some other error - tmpLog.warning('Failed to copy {0}: {1}'.format(remoteJsonFilePath, str(status))) - return {} - - try: - with open(localJsonFilePath) as jsonFile: - retDict = json.load(jsonFile) - os.remove(localJsonFilePath) - except Exception: - tmpLog.debug('failed to load json') - return {} - tmpLog.debug('found') - return retDict - - - def feed_events(self, workspec, events_dict): - '''Havester has an event range to pass to job''' - - # get logger - arclog = arc_utils.ARCLogger(baselogger, workspec.workerID) - tmpLog = arclog.log - - # Upload to jobid/jsonEventsFeedFileName, delete jobid/jsonEventsRequestFileName - job = workspec.workAttributes['arcjob'] - arcid = job['JobID'] - # Set certificate to use for interacting with ARC CE - usercfg = arc.UserConfig(self.cred_type) - if not self._setup_proxy(usercfg, workspec, arcid, tmpLog): - return False - - retVal = True - if workspec.mapType in [WorkSpec.MT_OneToOne, WorkSpec.MT_MultiWorkers]: - # put the json just under the access point then upload to ARC CE - localJsonFilePath = os.path.join(workspec.get_access_point(), jsonEventsFeedFileName) - tmpLog.debug('feeding events to {0}'.format(localJsonFilePath)) - try: - with open(localJsonFilePath, 'w') as jsonFile: - json.dump(events_dict, jsonFile) - except Exception: - core_utils.dump_error_message(tmpLog) - retVal = False - - remoteJsonFilePath = '%s/%s' % (arcid, jsonEventsFeedFileName) - # Try to copy the file - status = self._copy_file(localJsonFilePath, remoteJsonFilePath, usercfg, tmpLog) - if not status: - tmpLog.error('Failed to feed events to {0}: {1}'.format(remoteJsonFilePath, str(status))) - retVal = False - else: - remoteJsonEventsRequestFile = '%s/%s' % (arcid, jsonEventsRequestFileName) - status = self._delete_file(remoteJsonEventsRequestFile, usercfg, tmpLog) - if not status and status.GetErrno() != errno.ENOENT: - tmpLog.error('Failed to delete event request file at {0}'.format(remoteJsonEventsRequestFile)) - - elif workspec.mapType == WorkSpec.MT_MultiJobs: - # TOBEFIXED - pass - # remove request file - try: - jsonFilePath = os.path.join(workspec.get_access_point(), jsonEventsFeedFileName) - os.remove(jsonFilePath) - except Exception: - pass - tmpLog.debug('done') - return retVal - - - def events_to_update(self, workspec): - '''Report events processed for harvester to update''' - - # get logger - arclog = arc_utils.ARCLogger(baselogger, workspec.workerID) - tmpLog = arclog.log - - job = workspec.workAttributes['arcjob'] - arcid = job['JobID'] - # Set certificate to use for interacting with ARC CE - usercfg = arc.UserConfig(self.cred_type) - if not self._setup_proxy(usercfg, workspec, arcid, tmpLog): - return False - - # Check for jobid/jsonEventsUpdateFileName on CE, rename to .read - retDict = dict() - for pandaID in workspec.pandaid_list: - - # first look for json.read which is not yet acknowledged - accessPoint = self.get_access_point(workspec, pandaID) - localJsonFilePath = os.path.join(accessPoint, jsonEventsUpdateFileName) - remoteJsonFilePathRead = '%s/%s%s' % (arcid, jsonEventsUpdateFileName, suffixReadJson) - tmpLog.debug('looking for event update file {0}'.format(remoteJsonFilePathRead)) - - status = self._copy_file(remoteJsonFilePathRead, localJsonFilePath, usercfg, tmpLog) - if not status: - if status.GetErrno() != errno.ENOENT: - tmpLog.warning('Failed checking {0}: {1}'.format(remoteJsonFilePathRead, str(status))) - continue - - # Look for new json - remoteJsonFilePath = '%s/%s' % (arcid, jsonEventsUpdateFileName) - status = self._copy_file(remoteJsonFilePath, localJsonFilePath, usercfg, tmpLog) - if not status: - if status.GetErrno() != errno.ENOENT: - tmpLog.warning('Failed checking {0}: {1}'.format(remoteJsonFilePath, str(status))) - else: - # not found - tmpLog.debug('not found') - continue - - # Rename to prevent from being overwritten - # Gridftp does not support renaming so upload .read file and delete old one - status = self._copy_file(localJsonFilePath, remoteJsonFilePathRead, usercfg, tmpLog) - if not status: - tmpLog.warning('Failed copying {0} to {1}: {2}'.format(localJsonFilePath, remoteJsonFilePathRead, str(status))) - # If rename fails, delete old file anyway - status = self._delete_file(remoteJsonFilePath, usercfg, tmpLog) - if not status: - tmpLog.warning('Failed deleting {0}: {1}'.format(remoteJsonFilePath, str(status))) - - # load json - nData = 0 - try: - with open(localJsonFilePath) as jsonFile: - tmpOrigDict = json.load(jsonFile) - newDict = dict() - # change the key from str to int - for tmpPandaID, tmpDict in tmpOrigDict.iteritems(): - tmpPandaID = long(tmpPandaID) - retDict[tmpPandaID] = tmpDict - nData += 1 - except Exception: - raise - tmpLog.error('failed to load json') - # delete local file - try: - os.remove(localJsonFilePath) - except Exception: - pass - tmpLog.debug('got {0} events for PandaID={1}'.format(nData, pandaID)) - return retDict - - - def acknowledge_events_files(self, workspec): - '''Tell workers that harvester received events/files''' - - # get logger - arclog = arc_utils.ARCLogger(baselogger, workspec.workerID) - tmpLog = arclog.log - - job = workspec.workAttributes['arcjob'] - arcid = job['JobID'] - # Set certificate to use for interacting with ARC CE - usercfg = arc.UserConfig(self.cred_type) - if not self._setup_proxy(usercfg, workspec, arcid, tmpLog): - return False - - # Delete jobid/jsonEventsUpdateFileName.read - for pandaID in workspec.pandaid_list: - accessPoint = self.get_access_point(workspec, pandaID) - remoteJsonFilePath = '%s/%s%s' % (arcid, jsonEventsUpdateFileName, suffixReadJson) - status = self._delete_file(remoteJsonFilePath, usercfg, tmpLog) - if not status and status.GetErrno() != errno.ENOENT: - tmpLog.error('Failed deleting {0}: {1}'.format(remoteJsonFilePath, str(status))) - - tmpLog.debug('done') - return - - - # The remaining methods do not apply to ARC - def feed_jobs(self, workspec, jobspec_list): - '''Pass job to worker. No-op for Grid''' - return True - - def get_files_to_stage_out(self, workspec): - '''Not required in Grid case''' - return {} - - def job_requested(self, workspec): - '''Used in pull model to say that worker is ready for a job''' - return False - - def setup_access_points(self, workspec_list): - '''Access is through CE so nothing to set up here''' - pass - - def get_panda_ids(self, workspec): - '''For pull model, get panda IDs assigned to jobs''' - return [] - - def kill_requested(self, workspec): - '''Worker wants to kill itself (?)''' - return False - - def is_alive(self, workspec, time_limit): - '''Check if worker is alive, not for Grid''' - return True - - -def test(): - from pandaharvester.harvestercore.work_spec import WorkSpec - wspec = WorkSpec() - jobid = "gsiftp://pcoslo5.cern.ch:2811/jobs/XkNNDmultdtn1ZPzno6AuCjpABFKDmABFKDmwqyLDmABFKDm8dOcOn" - wspec.batchID = jobid - workAttributes = {"arcjob": {}} - workAttributes["arcjob"]["JobID"] = wspec.batchID - workAttributes["arcjob"]["JobStatusURL"] = "ldap://{0}:2135/mds-vo-name=local,o=grid??sub?(nordugrid-job-globalid={1})".format(urlparse(jobid).netloc, jobid) - workAttributes["arcjob"]["JobStatusInterfaceName"] = "org.nordugrid.ldapng" - jobmanagementurl = arc.URL(wspec.batchID) - jobmanagementurl.ChangePath("/jobs") - workAttributes["arcjob"]["JobManagementURL"] = jobmanagementurl.str() - workAttributes["arcjob"]["JobManagementInterfaceName"] = "org.nordugrid.gridftpjob" - workAttributes["proxyrole"] = 'production' - - wspec.workAttributes = workAttributes - wspec.accessPoint = '/tmp' - wspec.mapType = WorkSpec.MT_OneToOne - wspec.pandaid_list = [1234] - print wspec.workAttributes - - messenger = ARCMessenger() - print messenger.events_requested(wspec) - print messenger.feed_events(wspec, {'event': 1234}) - print messenger.events_to_update(wspec) - messenger.acknowledge_events_files(wspec) - -if __name__ == '__main__': - test() diff --git a/pandaharvester/harvestermessenger/http_server_messenger.py b/pandaharvester/harvestermessenger/http_server_messenger.py index 398e3dbf..94bf359c 100644 --- a/pandaharvester/harvestermessenger/http_server_messenger.py +++ b/pandaharvester/harvestermessenger/http_server_messenger.py @@ -2,13 +2,8 @@ import os import os.path import threading - -try: - from json.decoder import JSONDecodeError -except ImportError: - JSONDecodeError = ValueError - from http.server import BaseHTTPRequestHandler, HTTPServer +from json.decoder import JSONDecodeError from queue import Queue # try: diff --git a/pandaharvester/harvestermessenger/shared_file_messenger.py b/pandaharvester/harvestermessenger/shared_file_messenger.py index 286c9315..65722993 100644 --- a/pandaharvester/harvestermessenger/shared_file_messenger.py +++ b/pandaharvester/harvestermessenger/shared_file_messenger.py @@ -1,41 +1,20 @@ import copy import datetime +import fnmatch import itertools import json -import os -import shutil -import tarfile - -try: - from urllib.parse import urlencode -except ImportError: - from urllib import urlencode - -try: - import subprocess32 as subprocess -except ImportError: - import subprocess - -try: - from os import scandir, walk -except ImportError: - from scandir import scandir, walk - -try: - from shutil import which -except ImportError: - # before python 3.3 - from distutils.spawn import find_executable as which - -import fnmatch import multiprocessing +import os import os.path import re +import shutil +import subprocess +import tarfile import uuid from concurrent.futures import ThreadPoolExecutor as Pool - -from future.utils import iteritems -from past.builtins import long +from os import scandir, walk +from shutil import which +from urllib.parse import urlencode from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils @@ -350,8 +329,8 @@ def get_files_to_stage_out(self, workspec): sizeMap = dict() chksumMap = dict() eventsList = dict() - for tmpPandaID, tmpEventMapList in iteritems(loadDict): - tmpPandaID = long(tmpPandaID) + for tmpPandaID, tmpEventMapList in loadDict.items(): + tmpPandaID = int(tmpPandaID) # test if tmpEventMapList is a list if not isinstance(tmpEventMapList, list): tmpLog.error("loaded data item is not a list") @@ -591,8 +570,8 @@ def events_to_update(self, workspec): tmpOrigDict = json.load(jsonFile) newDict = dict() # change the key from str to int - for tmpPandaID, tmpDict in iteritems(tmpOrigDict): - tmpPandaID = long(tmpPandaID) + for tmpPandaID, tmpDict in tmpOrigDict.items(): + tmpPandaID = int(tmpPandaID) retDict[tmpPandaID] = tmpDict nData += len(tmpDict) except Exception: @@ -617,14 +596,14 @@ def acknowledge_events_files(self, workspec): try: jsonFilePath = os.path.join(accessPoint, jsonEventsUpdateFileName) jsonFilePath += suffixReadJson - jsonFilePath_rename = jsonFilePath + "." + datetime.datetime.utcnow().strftime("%Y-%m-%d_%H_%M_%S.%f") + jsonFilePath_rename = jsonFilePath + "." + core_utils.naive_utcnow().strftime("%Y-%m-%d_%H_%M_%S.%f") os.rename(jsonFilePath, jsonFilePath_rename) except Exception: pass try: jsonFilePath = os.path.join(accessPoint, jsonOutputsFileName) jsonFilePath += suffixReadJson - jsonFilePath_rename = jsonFilePath + "." + datetime.datetime.utcnow().strftime("%Y-%m-%d_%H_%M_%S.%f") + jsonFilePath_rename = jsonFilePath + "." + core_utils.naive_utcnow().strftime("%Y-%m-%d_%H_%M_%S.%f") os.rename(jsonFilePath, jsonFilePath_rename) except Exception: pass @@ -765,8 +744,8 @@ def post_processing(self, workspec, jobspec_list, map_type): if "merged" in tmpData: output_lfns = set() fileDict.setdefault(jobSpec.PandaID, []) - for tmpIn, tmpOuts in iteritems(tmpData["merged"]): - for tmpLFN, tmpFileDict in iteritems(tmpOuts): + for tmpIn, tmpOuts in tmpData["merged"].items(): + for tmpLFN, tmpFileDict in tmpOuts.items(): if tmpLFN in output_lfns: continue output_lfns.add(tmpLFN) @@ -846,12 +825,12 @@ def is_alive(self, workspec, time_limit): jsonFilePath = os.path.join(workspec.get_access_point(), heartbeatFile) tmpLog.debug(f"looking for heartbeat file {jsonFilePath}") if not os.path.exists(jsonFilePath): # no heartbeat file was found - tmpLog.debug(f"startTime: {workspec.startTime}, now: {datetime.datetime.utcnow()}") + tmpLog.debug(f"startTime: {workspec.startTime}, now: {core_utils.naive_utcnow()}") if not workspec.startTime: # the worker didn't even have time to start tmpLog.debug("heartbeat not found, but no startTime yet for worker") return True - elif datetime.datetime.utcnow() - workspec.startTime < datetime.timedelta(minutes=time_limit): + elif core_utils.naive_utcnow() - workspec.startTime < datetime.timedelta(minutes=time_limit): # the worker is too young and maybe didn't have time to generate the heartbeat tmpLog.debug("heartbeat not found, but worker too young") return True @@ -860,9 +839,9 @@ def is_alive(self, workspec, time_limit): tmpLog.debug("not found") return None try: - mtime = datetime.datetime.utcfromtimestamp(os.path.getmtime(jsonFilePath)) + mtime = core_utils.naive_utcfromtimestamp(os.path.getmtime(jsonFilePath)) tmpLog.debug(f"last modification time : {mtime}") - if datetime.datetime.utcnow() - mtime > datetime.timedelta(minutes=time_limit): + if core_utils.naive_utcnow() - mtime > datetime.timedelta(minutes=time_limit): tmpLog.debug("too old") return False tmpLog.debug("OK") diff --git a/pandaharvester/harvestermiddleware/direct_ssh_herder.py b/pandaharvester/harvestermiddleware/direct_ssh_herder.py index d911cccd..2c5be82e 100644 --- a/pandaharvester/harvestermiddleware/direct_ssh_herder.py +++ b/pandaharvester/harvestermiddleware/direct_ssh_herder.py @@ -1,7 +1,6 @@ import json import types -import six from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase @@ -77,7 +76,7 @@ def __call__(self, *args, **kwargs): "args": core_utils.pickle_to_text(args), "kwargs": core_utils.pickle_to_text(kwargs), } - stdout, stderr = self.conn.communicate(input=six.b(json.dumps(params))) + stdout, stderr = self.conn.communicate(input=json.dumps(params).encode("latin_1")) if self.conn.returncode == 0: return_dict = json.loads(stdout) if "exception" in return_dict: diff --git a/pandaharvester/harvestermiddleware/ssh_master_pool.py b/pandaharvester/harvestermiddleware/ssh_master_pool.py index bfdce140..cb12874f 100644 --- a/pandaharvester/harvestermiddleware/ssh_master_pool.py +++ b/pandaharvester/harvestermiddleware/ssh_master_pool.py @@ -1,24 +1,16 @@ import os import random +import subprocess import tempfile import threading import time import uuid import pexpect -import six - -try: - import subprocess32 as subprocess -except Exception: - import subprocess from pandaharvester.harvestercore import core_utils -if six.PY2: - pexpect_spawn = pexpect.spawn -else: - pexpect_spawn = pexpect.spawnu +pexpect_spawn = pexpect.spawnu # logger baseLogger = core_utils.setup_logger("ssh_master_pool") diff --git a/pandaharvester/harvestermiddleware/ssh_tunnel_pool.py b/pandaharvester/harvestermiddleware/ssh_tunnel_pool.py index 89348982..ee435cde 100644 --- a/pandaharvester/harvestermiddleware/ssh_tunnel_pool.py +++ b/pandaharvester/harvestermiddleware/ssh_tunnel_pool.py @@ -4,13 +4,10 @@ import uuid import pexpect -import six + from pandaharvester.harvestercore import core_utils -if six.PY2: - pexpect_spawn = pexpect.spawn -else: - pexpect_spawn = pexpect.spawnu +pexpect_spawn = pexpect.spawnu # logger baseLogger = core_utils.setup_logger("ssh_tunnel_pool") diff --git a/pandaharvester/harvestermisc/htcondor_utils.py b/pandaharvester/harvestermisc/htcondor_utils.py index ac64d23c..3587f3fb 100644 --- a/pandaharvester/harvestermisc/htcondor_utils.py +++ b/pandaharvester/harvestermisc/htcondor_utils.py @@ -4,23 +4,14 @@ import multiprocessing import random import re +import subprocess import tempfile import threading import time import traceback import xml.etree.ElementTree as ET +from threading import get_ident -try: - import subprocess32 as subprocess -except Exception: - import subprocess - -try: - from threading import get_ident -except ImportError: - from thread import get_ident - -import six from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.core_utils import SingletonWithID @@ -212,7 +203,7 @@ def condor_submit_process(mp_queue, host, jdl_map_list): # Condor queue cache fifo -class CondorQCacheFifo(six.with_metaclass(SingletonWithID, SpecialFIFOBase)): +class CondorQCacheFifo(SpecialFIFOBase, metaclass=SingletonWithID): global_lock_id = -1 def __init__(self, target, *args, **kwargs): @@ -364,7 +355,7 @@ def renew_session(self, retry=3, init=False): # Condor job query -class CondorJobQuery(six.with_metaclass(SingletonWithID, CondorClient)): +class CondorJobQuery(CondorClient, metaclass=SingletonWithID): # class lock classLock = threading.Lock() # Query commands @@ -671,7 +662,7 @@ def cleanup_cache(timeout=60): # Condor job submit -class CondorJobSubmit(six.with_metaclass(SingletonWithID, CondorClient)): +class CondorJobSubmit(CondorClient, metaclass=SingletonWithID): # class lock classLock = threading.Lock() @@ -822,7 +813,7 @@ def submit_with_python_process(self, jdl_list, use_spool=False): # Condor job remove -class CondorJobManage(six.with_metaclass(SingletonWithID, CondorClient)): +class CondorJobManage(CondorClient, metaclass=SingletonWithID): # class lock classLock = threading.Lock() diff --git a/pandaharvester/harvestermisc/idds_utils.py b/pandaharvester/harvestermisc/idds_utils.py index 35656065..d1a81803 100644 --- a/pandaharvester/harvestermisc/idds_utils.py +++ b/pandaharvester/harvestermisc/idds_utils.py @@ -1,12 +1,8 @@ import os +import subprocess import requests -try: - import subprocess32 as subprocess -except ImportError: - import subprocess - # get HP point def get_hp_point(idds_url, task_id, point_id, tmp_log, verbose): diff --git a/pandaharvester/harvestermisc/info_utils.py b/pandaharvester/harvestermisc/info_utils.py index 1d6d4c44..b4af0350 100644 --- a/pandaharvester/harvestermisc/info_utils.py +++ b/pandaharvester/harvestermisc/info_utils.py @@ -1,9 +1,6 @@ import threading import time -import six -from future.utils import iteritems - from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore.core_utils import SingletonWithID from pandaharvester.harvestercore.db_interface import DBInterface @@ -13,7 +10,7 @@ resolver_config = getattr(harvester_config.qconf, "resolverConfig", {}) -class PandaQueuesDict(six.with_metaclass(SingletonWithID, dict, PluginBase)): +class PandaQueuesDict(dict, PluginBase, metaclass=SingletonWithID): """ Dictionary of PanDA queue info from DB by cacher Key is PanDA Resource name (rather than PanDA Queue name) @@ -43,7 +40,7 @@ def _refresh(self): panda_queues_cache = self.dbInterface.get_cache(self.cacher_key) if panda_queues_cache and isinstance(panda_queues_cache.data, dict): panda_queues_dict = panda_queues_cache.data - for k, v in iteritems(panda_queues_dict): + for k, v in panda_queues_dict.items(): try: panda_resource = v["panda_resource"] assert k == v["nickname"] @@ -107,7 +104,7 @@ def get_queue_status(self, panda_resource): @to_refresh def get_all_queue_names(self): names = set() - for queue_name, queue_dict in iteritems(self): + for queue_name, queue_dict in self.items(): if queue_dict.get("pilot_manager") in ["Harvester"] and queue_dict.get("harvester") == harvesterID: names.add(queue_name) return names diff --git a/pandaharvester/harvestermisc/rucio_utils.py b/pandaharvester/harvestermisc/rucio_utils.py index 76ea57c8..8ad54658 100644 --- a/pandaharvester/harvestermisc/rucio_utils.py +++ b/pandaharvester/harvestermisc/rucio_utils.py @@ -2,10 +2,7 @@ utilities routines associated with Rucio CLI access """ -try: - import subprocess32 as subprocess -except BaseException: - import subprocess +import subprocess from pandaharvester.harvestercore import core_utils diff --git a/pandaharvester/harvestermisc/titan_utils.py b/pandaharvester/harvestermisc/titan_utils.py index 922e64a2..e77da9a1 100644 --- a/pandaharvester/harvestermisc/titan_utils.py +++ b/pandaharvester/harvestermisc/titan_utils.py @@ -1,14 +1,10 @@ +import datetime +import subprocess + from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestercore.work_spec import WorkSpec as ws -try: - import subprocess32 as subprocess -except Exception: - import subprocess - -import datetime - # logger baseLogger = core_utils.setup_logger("titan_utils") diff --git a/pandaharvester/harvestermisc/token_utils.py b/pandaharvester/harvestermisc/token_utils.py index 72b133e2..56292805 100644 --- a/pandaharvester/harvestermisc/token_utils.py +++ b/pandaharvester/harvestermisc/token_utils.py @@ -4,7 +4,6 @@ import time import requests -import six def _md5sum(data): @@ -12,7 +11,7 @@ def _md5sum(data): get md5sum hexadecimal string of data """ hash = hashlib.md5() - hash.update(six.b(data)) + hash.update(str(data).encode("latin_1")) hash_hex = hash.hexdigest() return hash_hex diff --git a/pandaharvester/harvestermonitor/cobalt_monitor.py b/pandaharvester/harvestermonitor/cobalt_monitor.py index 0e9a385e..c33c6421 100644 --- a/pandaharvester/harvestermonitor/cobalt_monitor.py +++ b/pandaharvester/harvestermonitor/cobalt_monitor.py @@ -1,12 +1,7 @@ -import re - -try: - import subprocess32 as subprocess -except BaseException: - import subprocess - import json import os.path +import re +import subprocess from pprint import pprint from pandaharvester.harvestercore import core_utils diff --git a/pandaharvester/harvestermonitor/htcondor_monitor.py b/pandaharvester/harvestermonitor/htcondor_monitor.py index acee1c04..2b1693c5 100644 --- a/pandaharvester/harvestermonitor/htcondor_monitor.py +++ b/pandaharvester/harvestermonitor/htcondor_monitor.py @@ -2,8 +2,6 @@ import time from concurrent.futures import ThreadPoolExecutor as Pool -import six - from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.pilot_errors import PilotErrors @@ -114,7 +112,7 @@ def _check_one_worker(workspec, job_ads_all_dict, cancel_unknown=False, held_tim tmpLog.debug(f"trying to kill job submissionHost={workspec.submissionHost} batchID={workspec.batchID} due to HoldReason: {hold_reason}") else: tmpLog.debug(f"trying to kill job submissionHost={workspec.submissionHost} batchID={workspec.batchID} due to held too long") - for submissionHost, batchIDs_list in six.iteritems(get_host_batchid_map([workspec])): + for submissionHost, batchIDs_list in get_host_batchid_map([workspec]).items(): condor_job_manage = CondorJobManage(id=workspec.submissionHost) try: ret_map = condor_job_manage.remove(batchIDs_list) @@ -251,7 +249,7 @@ def check_workers(self, workspec_list): tmpLog.debug("start") # Loop over submissionHost job_ads_all_dict = {} - for submissionHost, batchIDs_list in six.iteritems(get_host_batchid_map(workspec_list)): + for submissionHost, batchIDs_list in get_host_batchid_map(workspec_list).items(): # Record batch job query result to this dict, with key = batchID try: job_query = CondorJobQuery( @@ -312,7 +310,7 @@ def report_updated_workers(self, time_window): tmpLog.error(ret_err_str) # Choose workers updated within a time window workers_to_check_list = [] - for condor_job_id, job_ads in six.iteritems(job_ads_all_dict): + for condor_job_id, job_ads in job_ads_all_dict.items(): # put in worker cache fifo, with lock mechanism job_EnteredCurrentStatus = job_ads.get("EnteredCurrentStatus") if not (job_EnteredCurrentStatus > timeNow - time_window): diff --git a/pandaharvester/harvestermonitor/k8s_monitor.py b/pandaharvester/harvestermonitor/k8s_monitor.py index 907125e1..0f1b64ef 100644 --- a/pandaharvester/harvestermonitor/k8s_monitor.py +++ b/pandaharvester/harvestermonitor/k8s_monitor.py @@ -8,7 +8,6 @@ from pandaharvester.harvestercore.worker_errors import WorkerErrors from pandaharvester.harvestermisc.info_utils_k8s import PandaQueuesDictK8s from pandaharvester.harvestermisc.k8s_utils import k8s_Client - from pandaharvester.harvestermonitor.monitor_common import get_payload_errstr_from_ec base_logger = core_utils.setup_logger("k8s_monitor") @@ -135,7 +134,7 @@ def check_a_worker(self, workspec): # initialization job_id = workspec.batchID error_message = "" - time_now = datetime.datetime.utcnow() + time_now = core_utils.naive_utcnow() pods_status_list = [] pods_status_message_list = [] pods_name_to_delete_list = [] @@ -199,7 +198,9 @@ def check_a_worker(self, workspec): elif pods_status_list: # we found pods belonging to our job. Obtain the final status tmp_log.debug(f"pods_status_list={pods_status_list}") - new_status, exit_code, sub_msg = self.check_pods_status(pods_status_list, pods_status_message_list, containers_state_list, containers_exit_code_list) + new_status, exit_code, sub_msg = self.check_pods_status( + pods_status_list, pods_status_message_list, containers_state_list, containers_exit_code_list + ) if sub_msg: error_message += sub_msg tmp_log.debug(f"new_status={new_status}, error_message={error_message}") diff --git a/pandaharvester/harvestermonitor/lsf_monitor.py b/pandaharvester/harvestermonitor/lsf_monitor.py index 97f27898..cb050b25 100644 --- a/pandaharvester/harvestermonitor/lsf_monitor.py +++ b/pandaharvester/harvestermonitor/lsf_monitor.py @@ -1,11 +1,7 @@ import re +import subprocess from shlex import quote, split -try: - import subprocess32 as subprocess -except BaseException: - import subprocess - from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestercore.work_spec import WorkSpec diff --git a/pandaharvester/harvestermonitor/pbs_monitor.py b/pandaharvester/harvestermonitor/pbs_monitor.py index 9a59cf2b..7e7618da 100644 --- a/pandaharvester/harvestermonitor/pbs_monitor.py +++ b/pandaharvester/harvestermonitor/pbs_monitor.py @@ -1,9 +1,5 @@ import re - -try: - import subprocess32 as subprocess -except BaseException: - import subprocess +import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase diff --git a/pandaharvester/harvestermonitor/slurm_bulk_monitor.py b/pandaharvester/harvestermonitor/slurm_bulk_monitor.py index 82b1865a..37139e22 100644 --- a/pandaharvester/harvestermonitor/slurm_bulk_monitor.py +++ b/pandaharvester/harvestermonitor/slurm_bulk_monitor.py @@ -1,7 +1,4 @@ -try: - import subprocess32 as subprocess -except ImportError: - import subprocess +import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase diff --git a/pandaharvester/harvestermonitor/slurm_monitor.py b/pandaharvester/harvestermonitor/slurm_monitor.py index d674f051..14165007 100644 --- a/pandaharvester/harvestermonitor/slurm_monitor.py +++ b/pandaharvester/harvestermonitor/slurm_monitor.py @@ -1,9 +1,5 @@ import re - -try: - import subprocess32 as subprocess -except ImportError: - import subprocess +import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase diff --git a/pandaharvester/harvestermonitor/slurm_squeue_monitor.py b/pandaharvester/harvestermonitor/slurm_squeue_monitor.py index 89c163c9..1d3281d3 100644 --- a/pandaharvester/harvestermonitor/slurm_squeue_monitor.py +++ b/pandaharvester/harvestermonitor/slurm_squeue_monitor.py @@ -1,12 +1,7 @@ -import re - -try: - import subprocess32 as subprocess -except ImportError: - import subprocess - import json import os +import re +import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase diff --git a/pandaharvester/harvestermover/mover_utils.py b/pandaharvester/harvestermover/mover_utils.py index 60dd9064..3513aee7 100644 --- a/pandaharvester/harvestermover/mover_utils.py +++ b/pandaharvester/harvestermover/mover_utils.py @@ -1,12 +1,10 @@ import hashlib -import six - # construct file path def construct_file_path(base_path, scope, lfn): hash = hashlib.md5() - hash.update(six.b(f"{scope}:{lfn}")) + hash.update(f"{scope}:{lfn}".encode("latin_1")) hash_hex = hash.hexdigest() correctedscope = "/".join(scope.split(".")) dstURL = f"{base_path}/{correctedscope}/{hash_hex[0:2]}/{hash_hex[2:4]}/{lfn}" diff --git a/pandaharvester/harvesterpreparator/analysis_aux_preparator.py b/pandaharvester/harvesterpreparator/analysis_aux_preparator.py index e6daae59..f89c1e56 100644 --- a/pandaharvester/harvesterpreparator/analysis_aux_preparator.py +++ b/pandaharvester/harvesterpreparator/analysis_aux_preparator.py @@ -1,13 +1,10 @@ import os import shutil - -try: - import subprocess32 as subprocess -except Exception: - import subprocess +import subprocess import requests import requests.exceptions + from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestermover import mover_utils diff --git a/pandaharvester/harvesterpreparator/dummy_bulk_preparator.py b/pandaharvester/harvesterpreparator/dummy_bulk_preparator.py index 7d03b135..bd86378d 100644 --- a/pandaharvester/harvesterpreparator/dummy_bulk_preparator.py +++ b/pandaharvester/harvesterpreparator/dummy_bulk_preparator.py @@ -2,7 +2,6 @@ import threading import uuid -from future.utils import iteritems from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase @@ -65,7 +64,7 @@ def check_stage_in_status(self, jobspec): fileSpecs = self.dbInterface.get_files_with_group_id(self.dummy_transfer_id) # submit transfer if there are more than 10 files or the group was made before more than 10 min. # those thresholds may be config params. - if len(fileSpecs) >= 10 or groupUpdateTime < datetime.datetime.utcnow() - datetime.timedelta(minutes=10): + if len(fileSpecs) >= 10 or groupUpdateTime < core_utils.naive_utcnow() - datetime.timedelta(minutes=10): # submit transfer and get a real transfer ID # ... transferID = str(uuid.uuid4()) @@ -83,7 +82,7 @@ def check_stage_in_status(self, jobspec): # check transfer with real transfer IDs # ... # then update transfer status if successful - for transferID, transferInfo in iteritems(groups): + for transferID, transferInfo in groups.items(): jobspec.update_group_status_in_files(transferID, "done") return True, "" @@ -101,7 +100,7 @@ def resolve_input_paths(self, jobspec): # get input files inFiles = jobspec.get_input_file_attributes() # set path to each file - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): inFile["path"] = f"dummypath/{inLFN}" # set jobspec.set_input_file_paths(inFiles) diff --git a/pandaharvester/harvesterpreparator/dummy_preparator.py b/pandaharvester/harvesterpreparator/dummy_preparator.py index e8af4553..19f257c8 100644 --- a/pandaharvester/harvesterpreparator/dummy_preparator.py +++ b/pandaharvester/harvesterpreparator/dummy_preparator.py @@ -1,6 +1,5 @@ import uuid -from future.utils import iteritems from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase @@ -76,7 +75,7 @@ def check_stage_in_status(self, jobspec): # get groups of input files except ones already in ready state # transferGroups = jobspec.get_groups_of_input_files(skip_ready=True) # -- update transfer status - # for transferID, transferInfo in iteritems(transferGroups): + # for transferID, transferInfo in transferGroups.items(): # jobspec.update_group_status_in_files(transferID, 'done') return True, "" @@ -95,7 +94,7 @@ def resolve_input_paths(self, jobspec): # -- get input files inFiles = jobspec.get_input_file_attributes() # -- set path to each file - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): inFile["path"] = f"dummypath/{inLFN}" # -- set jobspec.set_input_file_paths(inFiles) diff --git a/pandaharvester/harvesterpreparator/go_bulk_preparator.py b/pandaharvester/harvesterpreparator/go_bulk_preparator.py index 5088d3c0..16cd5803 100644 --- a/pandaharvester/harvesterpreparator/go_bulk_preparator.py +++ b/pandaharvester/harvesterpreparator/go_bulk_preparator.py @@ -11,13 +11,13 @@ # TO BE REMOVED for python2.7 import requests.packages.urllib3 -from future.utils import iteritems from globus_sdk import ( NativeAppAuthClient, RefreshTokenAuthorizer, TransferClient, TransferData, ) + from pandaharvester.harvestermisc import globus_utils try: @@ -190,7 +190,7 @@ def check_stage_in_status(self, jobspec): ) tmpLog.debug(msgStr) # submit transfer if there are more than 10 files or the group was made before more than 10 min - if len(fileSpecs) >= 10 or groupUpdateTime < datetime.datetime.utcnow() - datetime.timedelta(minutes=10): + if len(fileSpecs) >= 10 or groupUpdateTime < core_utils.naive_utcnow() - datetime.timedelta(minutes=10): tmpLog.debug("prepare to transfer files") # submit transfer and get a real transfer ID # set the Globus destination Endpoint id and path will get them from Agis eventually @@ -451,7 +451,7 @@ def resolve_input_paths(self, jobspec): # get input files inFiles = jobspec.get_input_file_attributes() # set path to each file - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): inFile["path"] = mover_utils.construct_file_path(self.basePath, inFile["scope"], inLFN) # set jobspec.set_input_file_paths(inFiles) diff --git a/pandaharvester/harvesterpreparator/go_preparator.py b/pandaharvester/harvesterpreparator/go_preparator.py index e5ba90c4..bc939ffc 100644 --- a/pandaharvester/harvesterpreparator/go_preparator.py +++ b/pandaharvester/harvesterpreparator/go_preparator.py @@ -1,13 +1,13 @@ import os import sys -from future.utils import iteritems from globus_sdk import ( NativeAppAuthClient, RefreshTokenAuthorizer, TransferClient, TransferData, ) + from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestermisc import globus_utils @@ -138,7 +138,7 @@ def trigger_preparation(self, jobspec): files = [] lfns = [] inFiles = jobspec.get_input_file_attributes(skip_ready=True) - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): # set path to each file inFile["path"] = mover_utils.construct_file_path(self.basePath, inFile["scope"], inLFN) dstpath = inFile["path"] @@ -201,7 +201,7 @@ def resolve_input_paths(self, jobspec): # get input files inFiles = jobspec.get_input_file_attributes() # set path to each file - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): inFile["path"] = mover_utils.construct_file_path(self.basePath, inFile["scope"], inLFN) # set jobspec.set_input_file_paths(inFiles) diff --git a/pandaharvester/harvesterpreparator/gridftp_preparator.py b/pandaharvester/harvesterpreparator/gridftp_preparator.py index 831a8693..9bdbf60b 100644 --- a/pandaharvester/harvesterpreparator/gridftp_preparator.py +++ b/pandaharvester/harvesterpreparator/gridftp_preparator.py @@ -1,11 +1,7 @@ import os +import subprocess import tempfile -try: - import subprocess32 as subprocess -except Exception: - import subprocess - from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestermover import mover_utils diff --git a/pandaharvester/harvesterpreparator/pilotmover_mt_preparator.py b/pandaharvester/harvesterpreparator/pilotmover_mt_preparator.py index 480b2f2d..df0b79df 100644 --- a/pandaharvester/harvesterpreparator/pilotmover_mt_preparator.py +++ b/pandaharvester/harvesterpreparator/pilotmover_mt_preparator.py @@ -5,14 +5,14 @@ import time import traceback -from future.utils import iteritems -from pandaharvester.harvestercore import core_utils -from pandaharvester.harvestercore.plugin_base import PluginBase -from pandaharvester.harvestermover import mover_utils from pilot.api import data from pilot.info import infosys from pilot.info.filespec import FileSpec as PilotFileSpec +from pandaharvester.harvestercore import core_utils +from pandaharvester.harvestercore.plugin_base import PluginBase +from pandaharvester.harvestermover import mover_utils + # logger baseLogger = core_utils.setup_logger("pilotmover_mt_preparator") @@ -95,7 +95,7 @@ def trigger_preparation(self, jobspec): files = [] inFiles = jobspec.get_input_file_attributes(skip_ready=True) # set path to each file - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): inFile["path"] = mover_utils.construct_file_path(self.basePath, inFile["scope"], inLFN) tmpLog.debug(f"To check file: {inFile}") if os.path.exists(inFile["path"]): @@ -143,7 +143,7 @@ def trigger_preparation(self, jobspec): if file.status_code != 0: allChecked = False ErrMsg = ErrMsg + f" {file.lfn} " - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): if not os.path.isfile(inFile["path"]): allChecked = False ErrMsg = ErrMsg + f" {file.lfn} " @@ -163,7 +163,7 @@ def resolve_input_paths(self, jobspec): # get input files inFiles = jobspec.get_input_file_attributes() # set path to each file - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): inFile["path"] = mover_utils.construct_file_path(self.basePath, inFile["scope"], inLFN) # set jobspec.set_input_file_paths(inFiles) diff --git a/pandaharvester/harvesterpreparator/pilotmover_mt_preparator_kari.py b/pandaharvester/harvesterpreparator/pilotmover_mt_preparator_kari.py index a6a08857..8fbca0af 100644 --- a/pandaharvester/harvesterpreparator/pilotmover_mt_preparator_kari.py +++ b/pandaharvester/harvesterpreparator/pilotmover_mt_preparator_kari.py @@ -3,14 +3,14 @@ import threading import time -from future.utils import iteritems -from pandaharvester.harvestercore import core_utils -from pandaharvester.harvestercore.plugin_base import PluginBase -from pandaharvester.harvestermover import mover_utils from pilot.api import data from pilot.info import infosys from pilot.info.filespec import FileSpec as PilotFileSpec +from pandaharvester.harvestercore import core_utils +from pandaharvester.harvestercore.plugin_base import PluginBase +from pandaharvester.harvestermover import mover_utils + # logger baseLogger = core_utils.setup_logger("pilotmover_mt_preparator_kari") @@ -95,7 +95,7 @@ def trigger_preparation(self, jobspec): files = [] inFiles = jobspec.get_input_file_attributes(skip_ready=True) # set path to each file - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): inFile["path"] = mover_utils.construct_file_path(self.basePath, inFile["scope"], inLFN) tmpLog.debug(f"To check file: {inFile}") if os.path.exists(inFile["path"]): @@ -156,7 +156,7 @@ def trigger_preparation(self, jobspec): if file.status_code != 0: allChecked = False ErrMsg = ErrMsg + f" {file.lfn} " - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): if not os.path.isfile(inFile["path"]): allChecked = False ErrMsg = ErrMsg + f" {file.lfn} " @@ -175,7 +175,7 @@ def resolve_input_paths(self, jobspec): # get input files inFiles = jobspec.get_input_file_attributes() # set path to each file - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): inFile["path"] = mover_utils.construct_file_path(self.basePath, inFile["scope"], inLFN) # set jobspec.set_input_file_paths(inFiles) diff --git a/pandaharvester/harvesterpreparator/pilotmover_preparator.py b/pandaharvester/harvesterpreparator/pilotmover_preparator.py index cc74be49..14be353e 100644 --- a/pandaharvester/harvesterpreparator/pilotmover_preparator.py +++ b/pandaharvester/harvesterpreparator/pilotmover_preparator.py @@ -1,11 +1,11 @@ import os import os.path -from future.utils import iteritems +from pilot.api import data + from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestermover import mover_utils -from pilot.api import data # logger baseLogger = core_utils.setup_logger("pilotmover_preparator") @@ -47,7 +47,7 @@ def trigger_preparation(self, jobspec): inFiles = jobspec.get_input_file_attributes(skip_ready=True) # set path to each file tmpLog.info("Prepare files to download (construct path and verifiy existing files)") - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): inFile["path"] = mover_utils.construct_file_path(self.basePath, inFile["scope"], inLFN) # check if file exist. Skip alrady downoladed files if os.path.exists(inFile["path"]): @@ -96,7 +96,7 @@ def resolve_input_paths(self, jobspec): # get input files inFiles = jobspec.get_input_file_attributes() # set path to each file - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): inFile["path"] = mover_utils.construct_file_path(self.basePath, inFile["scope"], inLFN) # set jobspec.set_input_file_paths(inFiles) diff --git a/pandaharvester/harvesterpreparator/rse_direct_preparator.py b/pandaharvester/harvesterpreparator/rse_direct_preparator.py index 1b8079f8..65c61702 100644 --- a/pandaharvester/harvesterpreparator/rse_direct_preparator.py +++ b/pandaharvester/harvesterpreparator/rse_direct_preparator.py @@ -1,4 +1,3 @@ -from future.utils import iteritems from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestermover import mover_utils @@ -30,7 +29,7 @@ def resolve_input_paths(self, jobspec): # get input files inFiles = jobspec.get_input_file_attributes() # set path to each file - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): inFile["path"] = mover_utils.construct_file_path(self.basePath, inFile["scope"], inLFN) # set jobspec.set_input_file_paths(inFiles) diff --git a/pandaharvester/harvesterpreparator/rucio_preparator.py b/pandaharvester/harvesterpreparator/rucio_preparator.py index c084b051..f3b73784 100644 --- a/pandaharvester/harvesterpreparator/rucio_preparator.py +++ b/pandaharvester/harvesterpreparator/rucio_preparator.py @@ -3,7 +3,6 @@ import subprocess import traceback -from future.utils import iteritems from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestermover import mover_utils @@ -164,7 +163,7 @@ def resolve_input_paths(self, jobspec): # get input files inFiles = jobspec.get_input_file_attributes() # set path to each file - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): inFile["path"] = mover_utils.construct_file_path(base_dir, inFile["scope"], inLFN) # set jobspec.set_input_file_paths(inFiles) diff --git a/pandaharvester/harvesterpreparator/xrdcp_preparator.py b/pandaharvester/harvesterpreparator/xrdcp_preparator.py index 8c514712..ebc18e7d 100644 --- a/pandaharvester/harvesterpreparator/xrdcp_preparator.py +++ b/pandaharvester/harvesterpreparator/xrdcp_preparator.py @@ -1,11 +1,7 @@ import os +import subprocess import tempfile -try: - import subprocess32 as subprocess -except Exception: - import subprocess - from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase from pandaharvester.harvestermover import mover_utils diff --git a/pandaharvester/harvesterscripts/harvester_admin.py b/pandaharvester/harvesterscripts/harvester_admin.py index b6938598..6cb80a79 100644 --- a/pandaharvester/harvesterscripts/harvester_admin.py +++ b/pandaharvester/harvesterscripts/harvester_admin.py @@ -149,7 +149,7 @@ def _get_object_protective(i_index): def put_test(): sw.reset() multithread_executer(_put_object, n_object, n_thread) - sum_dict["put_time"] += sw.get_elapsed_time_in_sec(True) + sum_dict["put_time"] += sw.get_elapsed_time_in_sec() sum_dict["put_n"] += 1 print(f"Put {n_object} objects by {n_thread} threads" + sw.get_elapsed_time()) print(f"Now fifo size is {mq.size()}") @@ -157,21 +157,21 @@ def put_test(): def get_test(): sw.reset() multithread_executer(_get_object, n_object, n_thread) - sum_dict["get_time"] = sw.get_elapsed_time_in_sec(True) + sum_dict["get_time"] = sw.get_elapsed_time_in_sec() print(f"Get {n_object} objects by {n_thread} threads" + sw.get_elapsed_time()) print(f"Now fifo size is {mq.size()}") def get_protective_test(): sw.reset() multithread_executer(_get_object_protective, n_object, n_thread) - sum_dict["get_protective_time"] = sw.get_elapsed_time_in_sec(True) + sum_dict["get_protective_time"] = sw.get_elapsed_time_in_sec() print(f"Get {n_object} objects protective dequeue by {n_thread} threads" + sw.get_elapsed_time()) print(f"Now fifo size is {mq.size()}") def clear_test(): sw.reset() mq.fifo.clear() - sum_dict["clear_time"] = sw.get_elapsed_time_in_sec(True) + sum_dict["clear_time"] = sw.get_elapsed_time_in_sec() print("Cleared fifo" + sw.get_elapsed_time()) print(f"Now fifo size is {mq.size()}") diff --git a/pandaharvester/harvesterstager/dummy_bulk_stager.py b/pandaharvester/harvesterstager/dummy_bulk_stager.py index 18a7cc78..a557f434 100644 --- a/pandaharvester/harvesterstager/dummy_bulk_stager.py +++ b/pandaharvester/harvesterstager/dummy_bulk_stager.py @@ -46,7 +46,7 @@ def check_stage_out_status(self, jobspec): fileSpecs = self.dbInterface.get_files_with_group_id(dummy_transfer_id) # submit transfer if there are more than 10 files or the group was made before more than 10 min. # those thresholds may be config params. - if len(fileSpecs) >= 10 or groupUpdateTime < datetime.datetime.utcnow() - datetime.timedelta(minutes=10): + if len(fileSpecs) >= 10 or groupUpdateTime < core_utils.naive_utcnow() - datetime.timedelta(minutes=10): # submit transfer and get a real transfer ID # ... transferID = str(uuid.uuid4()) diff --git a/pandaharvester/harvesterstager/go_bulk_stager.py b/pandaharvester/harvesterstager/go_bulk_stager.py index 5ecb2fe8..9c145c5e 100644 --- a/pandaharvester/harvesterstager/go_bulk_stager.py +++ b/pandaharvester/harvesterstager/go_bulk_stager.py @@ -11,7 +11,6 @@ # TO BE REMOVED for python2.7 import requests.packages.urllib3 -from future.utils import iteritems from globus_sdk import ( NativeAppAuthClient, RefreshTokenAuthorizer, @@ -205,7 +204,7 @@ def check_stage_out_status(self, jobspec): # submit transfer if there are more than 10 files or the group was made before more than 10 min msgStr = f"dummy_transferID = {dummy_transferID} number of files = {len(fileSpecs)}" tmpLog.debug(msgStr) - if len(fileSpecs) >= 10 or groupUpdateTime < datetime.datetime.utcnow() - datetime.timedelta(minutes=10): + if len(fileSpecs) >= 10 or groupUpdateTime < core_utils.naive_utcnow() - datetime.timedelta(minutes=10): tmpLog.debug("prepare to transfer files") # submit transfer and get a real transfer ID # set the Globus destination Endpoint id and path will get them from Agis eventually @@ -466,7 +465,7 @@ def resolve_input_paths(self, jobspec): # get input files inFiles = jobspec.get_input_file_attributes() # set path to each file - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): inFile["path"] = mover_utils.construct_file_path(self.basePath, inFile["scope"], inLFN) # set jobspec.set_input_file_paths(inFiles) diff --git a/pandaharvester/harvesterstager/go_stager.py b/pandaharvester/harvesterstager/go_stager.py index e1982f60..e808eefc 100644 --- a/pandaharvester/harvesterstager/go_stager.py +++ b/pandaharvester/harvesterstager/go_stager.py @@ -6,7 +6,6 @@ # TO BE REMOVED for python2.7 import requests.packages.urllib3 -from future.utils import iteritems from globus_sdk import ( NativeAppAuthClient, RefreshTokenAuthorizer, @@ -266,7 +265,7 @@ def resolve_input_paths(self, jobspec): # get input files inFiles = jobspec.get_input_file_attributes() # set path to each file - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): inFile["path"] = mover_utils.construct_file_path(self.basePath, inFile["scope"], inLFN) # set jobspec.set_input_file_paths(inFiles) diff --git a/pandaharvester/harvesterstager/gridftp_stager.py b/pandaharvester/harvesterstager/gridftp_stager.py index ef92be5b..bc1bdf78 100644 --- a/pandaharvester/harvesterstager/gridftp_stager.py +++ b/pandaharvester/harvesterstager/gridftp_stager.py @@ -1,12 +1,8 @@ import os import re +import subprocess import tempfile -try: - import subprocess32 as subprocess -except Exception: - import subprocess - from pandaharvester.harvestercore import core_utils from pandaharvester.harvestermover import mover_utils diff --git a/pandaharvester/harvesterstager/rucio_stager.py b/pandaharvester/harvesterstager/rucio_stager.py index 7e35d2d9..c49a7fe5 100644 --- a/pandaharvester/harvesterstager/rucio_stager.py +++ b/pandaharvester/harvesterstager/rucio_stager.py @@ -4,12 +4,12 @@ import sys import uuid -from future.utils import iteritems -from pandaharvester.harvestercore import core_utils -from pandaharvester.harvestermover import mover_utils from rucio.client import Client as RucioClient from rucio.common.exception import RuleNotFound +from pandaharvester.harvestercore import core_utils +from pandaharvester.harvestermover import mover_utils + from .base_stager import BaseStager # logger @@ -124,7 +124,7 @@ def trigger_stage_out(self, jobspec): files[fileSpec.fileType].append(tmpFile) # loop over all file types to be registered to rucio rucioAPI = RucioClient() - for fileType, fileList in iteritems(files): + for fileType, fileList in files.items(): # set destination RSE if fileType in ["es_output", "zip_output"]: dstRSE = self.dstRSE_ES diff --git a/pandaharvester/harvesterstager/rucio_stager_hpc.py b/pandaharvester/harvesterstager/rucio_stager_hpc.py index b379b045..f5849ff9 100644 --- a/pandaharvester/harvesterstager/rucio_stager_hpc.py +++ b/pandaharvester/harvesterstager/rucio_stager_hpc.py @@ -1,9 +1,5 @@ -try: - import subprocess32 as subprocess -except ImportError: - import subprocess - import os +import subprocess import uuid from pandaharvester.harvestercore import core_utils diff --git a/pandaharvester/harvesterstager/rucio_stager_hpc_minikui.py b/pandaharvester/harvesterstager/rucio_stager_hpc_minikui.py index 284feed9..93e5bce7 100644 --- a/pandaharvester/harvesterstager/rucio_stager_hpc_minikui.py +++ b/pandaharvester/harvesterstager/rucio_stager_hpc_minikui.py @@ -1,10 +1,6 @@ -try: - import subprocess32 as subprocess -except ImportError: - import subprocess - import gc import os +import subprocess import uuid from concurrent.futures import ThreadPoolExecutor as Pool diff --git a/pandaharvester/harvesterstager/xrdcp_stager.py b/pandaharvester/harvesterstager/xrdcp_stager.py index 88dda6ec..5cf8f22d 100644 --- a/pandaharvester/harvesterstager/xrdcp_stager.py +++ b/pandaharvester/harvesterstager/xrdcp_stager.py @@ -1,12 +1,7 @@ import gc import os +import subprocess import tempfile - -try: - import subprocess32 as subprocess -except Exception: - import subprocess - import uuid from pandaharvester.harvestercore import core_utils diff --git a/pandaharvester/harvesterstager/yoda_rse_direct_stager.py b/pandaharvester/harvesterstager/yoda_rse_direct_stager.py index a58daa90..4b548dcc 100644 --- a/pandaharvester/harvesterstager/yoda_rse_direct_stager.py +++ b/pandaharvester/harvesterstager/yoda_rse_direct_stager.py @@ -12,7 +12,6 @@ import time import uuid -from future.utils import iteritems from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase diff --git a/pandaharvester/harvesterstager/yoda_rucio_rse_direct_stager.py b/pandaharvester/harvesterstager/yoda_rucio_rse_direct_stager.py index 1172f6c4..06b65209 100644 --- a/pandaharvester/harvesterstager/yoda_rucio_rse_direct_stager.py +++ b/pandaharvester/harvesterstager/yoda_rucio_rse_direct_stager.py @@ -12,13 +12,6 @@ import time import uuid -from future.utils import iteritems -from pandaharvester.harvesterconfig import harvester_config -from pandaharvester.harvestercore import core_utils -from pandaharvester.harvestercore.plugin_base import PluginBase -from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper -from pandaharvester.harvestermover import mover_utils -from pandaharvester.harvesterstager.base_stager import BaseStager from rucio.client import Client as RucioClient from rucio.common.exception import ( DataIdentifierAlreadyExists, @@ -27,6 +20,13 @@ FileAlreadyExists, ) +from pandaharvester.harvesterconfig import harvester_config +from pandaharvester.harvestercore import core_utils +from pandaharvester.harvestercore.plugin_base import PluginBase +from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper +from pandaharvester.harvestermover import mover_utils +from pandaharvester.harvesterstager.base_stager import BaseStager + from .base_stager import BaseStager diff --git a/pandaharvester/harvestersubmitter/cobalt_submitter.py b/pandaharvester/harvestersubmitter/cobalt_submitter.py index 69de7b48..ca991eb3 100644 --- a/pandaharvester/harvestersubmitter/cobalt_submitter.py +++ b/pandaharvester/harvestersubmitter/cobalt_submitter.py @@ -1,11 +1,7 @@ -import tempfile - -try: - import subprocess32 as subprocess -except BaseException: - import subprocess import os import stat +import subprocess +import tempfile from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase diff --git a/pandaharvester/harvestersubmitter/dummy_mcore_submitter.py b/pandaharvester/harvestersubmitter/dummy_mcore_submitter.py index 7ca7de7a..64875245 100644 --- a/pandaharvester/harvestersubmitter/dummy_mcore_submitter.py +++ b/pandaharvester/harvestersubmitter/dummy_mcore_submitter.py @@ -1,11 +1,6 @@ import os +import subprocess import uuid - -try: - import subprocess32 as subprocess -except BaseException: - import subprocess - from concurrent.futures import ProcessPoolExecutor as Pool from pandaharvester.harvestercore import core_utils diff --git a/pandaharvester/harvestersubmitter/htcondor_submitter.py b/pandaharvester/harvestersubmitter/htcondor_submitter.py index aa129ab4..2844d86c 100644 --- a/pandaharvester/harvestersubmitter/htcondor_submitter.py +++ b/pandaharvester/harvestersubmitter/htcondor_submitter.py @@ -531,7 +531,7 @@ def submit_workers(self, workspec_list): to_submit_any = True # get log subdirectory name from timestamp - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() log_subdir = timeNow.strftime("%y-%m-%d_%H") log_subdir_path = os.path.join(self.logDir, log_subdir) if self.condorSchedd is None or not self.useSpool: @@ -554,7 +554,7 @@ def submit_workers(self, workspec_list): # get queue info from CRIC by cacher in db if self.useCRIC: panda_queues_dict = PandaQueuesDict() - panda_queues_dict_last_refresh = datetime.datetime.utcfromtimestamp(panda_queues_dict.last_refresh_ts) + panda_queues_dict_last_refresh = core_utils.naive_utcfromtimestamp(panda_queues_dict.last_refresh_ts) tmpLog.debug(f"PandaQueuesDict last refresh at {panda_queues_dict_last_refresh}") panda_queue_name = panda_queues_dict.get_panda_queue_name(self.queueName) this_panda_queue_dict = panda_queues_dict.get(self.queueName, dict()) diff --git a/pandaharvester/harvestersubmitter/k8s_submitter.py b/pandaharvester/harvestersubmitter/k8s_submitter.py index 1cc283c6..bf762be5 100644 --- a/pandaharvester/harvestersubmitter/k8s_submitter.py +++ b/pandaharvester/harvestersubmitter/k8s_submitter.py @@ -1,12 +1,7 @@ import os import traceback - -try: - from urllib import unquote # Python 2.X -except ImportError: - from urllib.parse import unquote # Python 3+ - from concurrent.futures import ThreadPoolExecutor +from urllib.parse import unquote # Python 3+ from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils diff --git a/pandaharvester/harvestersubmitter/lsf_submitter.py b/pandaharvester/harvestersubmitter/lsf_submitter.py index faa729a7..4dedc175 100644 --- a/pandaharvester/harvestersubmitter/lsf_submitter.py +++ b/pandaharvester/harvestersubmitter/lsf_submitter.py @@ -1,12 +1,8 @@ import datetime import re +import subprocess import tempfile -try: - import subprocess32 as subprocess -except BaseException: - import subprocess - from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase diff --git a/pandaharvester/harvestersubmitter/pbs_submitter.py b/pandaharvester/harvestersubmitter/pbs_submitter.py index b7ff3613..4c56d2e1 100644 --- a/pandaharvester/harvestersubmitter/pbs_submitter.py +++ b/pandaharvester/harvestersubmitter/pbs_submitter.py @@ -1,11 +1,7 @@ import datetime +import subprocess import tempfile -try: - import subprocess32 as subprocess -except BaseException: - import subprocess - from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase diff --git a/pandaharvester/harvestersubmitter/slurm_submitter.py b/pandaharvester/harvestersubmitter/slurm_submitter.py index e6c7da13..f70bfdad 100644 --- a/pandaharvester/harvestersubmitter/slurm_submitter.py +++ b/pandaharvester/harvestersubmitter/slurm_submitter.py @@ -2,16 +2,10 @@ import os import re import stat +import subprocess import tempfile from math import ceil -import six - -try: - import subprocess32 as subprocess -except ImportError: - import subprocess - from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase @@ -85,7 +79,7 @@ def submit_workers(self, workspec_list): return retList def make_placeholder_map(self, workspec): - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() panda_queue_name = self.queueName this_panda_queue_dict = dict() @@ -154,7 +148,7 @@ def make_batch_script(self, workspec): template = f.read() tmpFile = tempfile.NamedTemporaryFile(delete=False, suffix="_submit.sh", dir=workspec.get_access_point()) placeholder = self.make_placeholder_map(workspec) - tmpFile.write(six.b(template.format_map(core_utils.SafeDict(placeholder)))) + tmpFile.write(str(template.format_map(core_utils.SafeDict(placeholder))).encode("latin_1")) tmpFile.close() # set execution bit and group permissions on the temp file diff --git a/pandaharvester/harvestersubmitter/slurm_submitter_jinja.py b/pandaharvester/harvestersubmitter/slurm_submitter_jinja.py index d68a0778..7debc35c 100644 --- a/pandaharvester/harvestersubmitter/slurm_submitter_jinja.py +++ b/pandaharvester/harvestersubmitter/slurm_submitter_jinja.py @@ -1,13 +1,8 @@ import re +import subprocess import tempfile import jinja2 -import six - -try: - import subprocess32 as subprocess -except ImportError: - import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase @@ -79,11 +74,11 @@ def make_batch_script(self, workspec): del tmpFile tmpFile = tempfile.NamedTemporaryFile(delete=False, suffix="_submit.sh", dir=workspec.get_access_point()) tmpFile.write( - six.b( + str( self.template.format( nCorePerNode=self.nCorePerNode, nNode=workspec.nCore // self.nCorePerNode, accessPoint=workspec.accessPoint, workerID=workspec.workerID ) - ) + ).encode("latin_1") ) tmpFile.close() return tmpFile.name @@ -99,7 +94,7 @@ def make_batch_script_jinja(self, workspec): tmpFile = tempfile.NamedTemporaryFile(delete=False, suffix="_submit.sh", dir=workspec.get_access_point()) tm = jinja2.Template(self.template) tmpFile.write( - six.b( + str( tm.render( nCorePerNode=self.nCorePerNode, nNode=workspec.nCore // self.nCorePerNode, @@ -107,18 +102,18 @@ def make_batch_script_jinja(self, workspec): workerID=workspec.workerID, workspec=workspec, ) - ) + ).encode("latin_1") ) - # tmpFile.write(six.b(self.template.format(nCorePerNode=self.nCorePerNode, + # tmpFile.write(str(self.template.format(nCorePerNode=self.nCorePerNode, # nNode=workspec.nCore // self.nCorePerNode, # accessPoint=workspec.accessPoint, - # workerID=workspec.workerID)) + # workerID=workspec.workerID)).encode("latin_1") # ) - # tmpFile.write(six.b(self.template.format(nCorePerNode=self.nCorePerNode, + # tmpFile.write(str(self.template.format(nCorePerNode=self.nCorePerNode, # nNode=workspec.nCore // self.nCorePerNode, # worker=workSpec, - # submitter=self)) + # submitter=self)).encode("latin_1") # ) tmpFile.close() return tmpFile.name diff --git a/pandaharvester/harvestersweeper/base_sweeper.py b/pandaharvester/harvestersweeper/base_sweeper.py index f62a98c7..292f1f97 100644 --- a/pandaharvester/harvestersweeper/base_sweeper.py +++ b/pandaharvester/harvestersweeper/base_sweeper.py @@ -1,11 +1,10 @@ import abc -import six from pandaharvester.harvestercore.plugin_base import PluginBase # dummy plugin for sweeper -class BaseSweeper(six.with_metaclass(abc.ABCMeta, PluginBase)): +class BaseSweeper(PluginBase, metaclass=abc.ABCMeta): # constructor def __init__(self, **kwarg): PluginBase.__init__(self, **kwarg) diff --git a/pandaharvester/harvestersweeper/cobalt_sweeper.py b/pandaharvester/harvestersweeper/cobalt_sweeper.py index 9868e4d8..75024ff3 100644 --- a/pandaharvester/harvestersweeper/cobalt_sweeper.py +++ b/pandaharvester/harvestersweeper/cobalt_sweeper.py @@ -1,17 +1,12 @@ # === Imports ================================================== import os +import shutil +import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase -try: - import subprocess32 as subprocess -except BaseException: - import subprocess - -import shutil - # ============================================================== # === Definitions ============================================== diff --git a/pandaharvester/harvestersweeper/gitlab_sweeper.py b/pandaharvester/harvestersweeper/gitlab_sweeper.py index 92258457..a041b081 100644 --- a/pandaharvester/harvestersweeper/gitlab_sweeper.py +++ b/pandaharvester/harvestersweeper/gitlab_sweeper.py @@ -1,13 +1,9 @@ import os import shutil +import subprocess import requests -try: - import subprocess32 as subprocess -except ImportError: - import subprocess - from pandaharvester.harvestercore import core_utils from pandaharvester.harvestermisc.gitlab_utils import get_job_params from pandaharvester.harvestersweeper.base_sweeper import BaseSweeper diff --git a/pandaharvester/harvestersweeper/htcondor_sweeper.py b/pandaharvester/harvestersweeper/htcondor_sweeper.py index 0575c2f8..2f284076 100644 --- a/pandaharvester/harvestersweeper/htcondor_sweeper.py +++ b/pandaharvester/harvestersweeper/htcondor_sweeper.py @@ -1,12 +1,6 @@ import os import shutil - -import six - -try: - import subprocess32 as subprocess -except Exception: - import subprocess +import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestermisc.htcondor_utils import ( @@ -94,7 +88,7 @@ def kill_workers(self, workspec_list): all_job_ret_map = {} retList = [] # Kill - for submissionHost, batchIDs_list in six.iteritems(get_host_batchid_map(workspec_list)): + for submissionHost, batchIDs_list in get_host_batchid_map(workspec_list).items(): try: condor_job_manage = CondorJobManage(id=submissionHost) ret_map = condor_job_manage.remove(batchIDs_list) diff --git a/pandaharvester/harvestersweeper/lsf_sweeper.py b/pandaharvester/harvestersweeper/lsf_sweeper.py index b729cbe8..1ec56477 100644 --- a/pandaharvester/harvestersweeper/lsf_sweeper.py +++ b/pandaharvester/harvestersweeper/lsf_sweeper.py @@ -1,10 +1,6 @@ import os import shutil - -try: - import subprocess32 as subprocess -except BaseException: - import subprocess +import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase diff --git a/pandaharvester/harvestersweeper/pbs_sweeper.py b/pandaharvester/harvestersweeper/pbs_sweeper.py index 90353a38..35e409d5 100644 --- a/pandaharvester/harvestersweeper/pbs_sweeper.py +++ b/pandaharvester/harvestersweeper/pbs_sweeper.py @@ -1,10 +1,6 @@ import os import shutil - -try: - import subprocess32 as subprocess -except BaseException: - import subprocess +import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase diff --git a/pandaharvester/harvestersweeper/slurm_sweeper.py b/pandaharvester/harvestersweeper/slurm_sweeper.py index e2013b8c..3a574634 100644 --- a/pandaharvester/harvestersweeper/slurm_sweeper.py +++ b/pandaharvester/harvestersweeper/slurm_sweeper.py @@ -1,10 +1,6 @@ import os import shutil - -try: - import subprocess32 as subprocess -except ImportError: - import subprocess +import subprocess from pandaharvester.harvestercore import core_utils from pandaharvester.harvestersweeper.base_sweeper import BaseSweeper diff --git a/pandaharvester/harvestertest/basicTest.py b/pandaharvester/harvestertest/basicTest.py index c2fd659a..39f8e119 100644 --- a/pandaharvester/harvestertest/basicTest.py +++ b/pandaharvester/harvestertest/basicTest.py @@ -3,7 +3,6 @@ import os import sys -from future.utils import iteritems from pandaharvester.harvesterconfig import harvester_config try: @@ -15,7 +14,7 @@ from pandaharvester.harvestercore.job_spec import JobSpec from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper -for loggerName, loggerObj in iteritems(logging.Logger.manager.loggerDict): +for loggerName, loggerObj in logging.Logger.manager.loggerDict.items(): if loggerName.startswith("panda.log"): if len(loggerObj.handlers) == 0: continue diff --git a/pandaharvester/harvestertest/cacherTest.py b/pandaharvester/harvestertest/cacherTest.py index 49c75309..cb9325b6 100644 --- a/pandaharvester/harvestertest/cacherTest.py +++ b/pandaharvester/harvestertest/cacherTest.py @@ -1,12 +1,11 @@ import logging import sys -from future.utils import iteritems from pandaharvester.harvesterbody.cacher import Cacher from pandaharvester.harvestercore.communicator_pool import CommunicatorPool from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy -for loggerName, loggerObj in iteritems(logging.Logger.manager.loggerDict): +for loggerName, loggerObj in logging.Logger.manager.loggerDict.items(): if loggerName.startswith("panda.log"): if len(loggerObj.handlers) == 0: continue diff --git a/pandaharvester/harvestertest/check_log_db_proxy_pool.py b/pandaharvester/harvestertest/check_log_db_proxy_pool.py index d17206b8..e780d4f0 100644 --- a/pandaharvester/harvestertest/check_log_db_proxy_pool.py +++ b/pandaharvester/harvestertest/check_log_db_proxy_pool.py @@ -3,7 +3,6 @@ import os import re -from future.utils import iteritems from pandalogger import logger_config logdir = logger_config.daemon["logdir"] @@ -26,7 +25,7 @@ # get average and longest lData = dict() aData = dict() -for method, exeTimes in iteritems(data): +for method, exeTimes in data.items(): exeTimes.sort() exeTimes.reverse() for exeTime in exeTimes[:nL]: diff --git a/pandaharvester/harvestertest/cleanDN.py b/pandaharvester/harvestertest/cleanDN.py index 156206af..850242a7 100644 --- a/pandaharvester/harvestertest/cleanDN.py +++ b/pandaharvester/harvestertest/cleanDN.py @@ -1,11 +1,7 @@ import re +import subprocess import sys -try: - import subprocess32 as subprocess -except BaseException: - import subprocess - def clean_user_id(id): try: diff --git a/pandaharvester/harvestertest/credMangerTest.py b/pandaharvester/harvestertest/credMangerTest.py index 20380425..9ab5834e 100644 --- a/pandaharvester/harvestertest/credMangerTest.py +++ b/pandaharvester/harvestertest/credMangerTest.py @@ -1,7 +1,6 @@ import logging import sys -from future.utils import iteritems from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_factory import PluginFactory @@ -52,7 +51,7 @@ def get_list(data): exeCores.append(exeCore) # setup logger to write to screen also -for loggerName, loggerObj in iteritems(logging.Logger.manager.loggerDict): +for loggerName, loggerObj in logging.Logger.manager.loggerDict.items(): if loggerName.startswith("panda.log"): if len(loggerObj.handlers) == 0: continue diff --git a/pandaharvester/harvestertest/further_testing_go_bulk_preparator-test.py b/pandaharvester/harvestertest/further_testing_go_bulk_preparator-test.py index cd4f2317..9268b12f 100644 --- a/pandaharvester/harvestertest/further_testing_go_bulk_preparator-test.py +++ b/pandaharvester/harvestertest/further_testing_go_bulk_preparator-test.py @@ -10,13 +10,13 @@ import time import uuid -from future.utils import iteritems from globus_sdk import ( NativeAppAuthClient, RefreshTokenAuthorizer, TransferClient, TransferData, ) + from pandaharvester.harvesterbody.cacher import Cacher from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils diff --git a/pandaharvester/harvestertest/further_testing_go_bulk_preparator.py b/pandaharvester/harvestertest/further_testing_go_bulk_preparator.py index 6c459afb..6d8201e7 100644 --- a/pandaharvester/harvestertest/further_testing_go_bulk_preparator.py +++ b/pandaharvester/harvestertest/further_testing_go_bulk_preparator.py @@ -10,13 +10,13 @@ import time import uuid -from future.utils import iteritems from globus_sdk import ( NativeAppAuthClient, RefreshTokenAuthorizer, TransferClient, TransferData, ) + from pandaharvester.harvesterbody.cacher import Cacher from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils diff --git a/pandaharvester/harvestertest/further_testing_go_bulk_stager.py b/pandaharvester/harvestertest/further_testing_go_bulk_stager.py index 0302ebfb..d8f9f87f 100644 --- a/pandaharvester/harvestertest/further_testing_go_bulk_stager.py +++ b/pandaharvester/harvestertest/further_testing_go_bulk_stager.py @@ -10,13 +10,13 @@ import time import uuid -from future.utils import iteritems from globus_sdk import ( NativeAppAuthClient, RefreshTokenAuthorizer, TransferClient, TransferData, ) + from pandaharvester.harvesterbody.cacher import Cacher from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils diff --git a/pandaharvester/harvestertest/getJobs.py b/pandaharvester/harvestertest/getJobs.py index 971af73c..8793aa77 100644 --- a/pandaharvester/harvestertest/getJobs.py +++ b/pandaharvester/harvestertest/getJobs.py @@ -5,11 +5,10 @@ import os import sys -from future.utils import iteritems from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper -for loggerName, loggerObj in iteritems(logging.Logger.manager.loggerDict): +for loggerName, loggerObj in logging.Logger.manager.loggerDict.items(): if loggerName.startswith("panda.log"): if len(loggerObj.handlers) == 0: continue diff --git a/pandaharvester/harvestertest/monitorFifoTest.py b/pandaharvester/harvestertest/monitorFifoTest.py index c96b62a1..d0e1e093 100644 --- a/pandaharvester/harvestertest/monitorFifoTest.py +++ b/pandaharvester/harvestertest/monitorFifoTest.py @@ -3,7 +3,6 @@ import sys import time -from future.utils import iteritems from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.fifos import MonitorFIFO from pandaharvester.harvestercore.job_spec import JobSpec diff --git a/pandaharvester/harvestertest/stageInTest_globus.py b/pandaharvester/harvestertest/stageInTest_globus.py index a169ab0d..3800f01c 100644 --- a/pandaharvester/harvestertest/stageInTest_globus.py +++ b/pandaharvester/harvestertest/stageInTest_globus.py @@ -10,13 +10,13 @@ import time import uuid -from future.utils import iteritems from globus_sdk import ( NativeAppAuthClient, RefreshTokenAuthorizer, TransferClient, TransferData, ) + from pandaharvester.harvesterbody.cacher import Cacher from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils @@ -64,7 +64,7 @@ def dump(obj): tmpLog = core_utils.make_logger(_logger, method_name="stageInTest_go_preparator") tmpLog.debug("start") -for loggerName, loggerObj in iteritems(logging.Logger.manager.loggerDict): +for loggerName, loggerObj in logging.Logger.manager.loggerDict.items(): # print "loggerName - {}".format(loggerName) if loggerName.startswith("panda.log"): if len(loggerObj.handlers) == 0: diff --git a/pandaharvester/harvestertest/stageOutTest_go_bulk_stager.py b/pandaharvester/harvestertest/stageOutTest_go_bulk_stager.py index 52e0d0a0..cc4b62b1 100644 --- a/pandaharvester/harvestertest/stageOutTest_go_bulk_stager.py +++ b/pandaharvester/harvestertest/stageOutTest_go_bulk_stager.py @@ -10,7 +10,6 @@ import time import uuid -from future.utils import iteritems from pandaharvester.harvesterbody.cacher import Cacher from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils diff --git a/pandaharvester/harvestertest/submitterTest.py b/pandaharvester/harvestertest/submitterTest.py index 85a0b093..e19e8bef 100644 --- a/pandaharvester/harvestertest/submitterTest.py +++ b/pandaharvester/harvestertest/submitterTest.py @@ -1,7 +1,6 @@ import os import sys -from future.utils import iteritems from pandaharvester.harvestercore.communicator_pool import CommunicatorPool from pandaharvester.harvestercore.job_spec import JobSpec from pandaharvester.harvestercore.plugin_factory import PluginFactory @@ -63,7 +62,7 @@ # set input file paths inFiles = jobSpec.get_input_file_attributes() - for inLFN, inFile in iteritems(inFiles): + for inLFN, inFile in inFiles.items(): inFile["path"] = f"{os.getcwd()}/{inLFN}" jobSpec.set_input_file_paths(inFiles) jobSpecList.append(jobSpec) diff --git a/pandaharvester/harvestertest/testCommunication.py b/pandaharvester/harvestertest/testCommunication.py index 7a3062c9..af81db93 100644 --- a/pandaharvester/harvestertest/testCommunication.py +++ b/pandaharvester/harvestertest/testCommunication.py @@ -1,10 +1,9 @@ import logging import sys -from future.utils import iteritems from pandaharvester.harvestercore.communicator_pool import CommunicatorPool -for loggerName, loggerObj in iteritems(logging.Logger.manager.loggerDict): +for loggerName, loggerObj in logging.Logger.manager.loggerDict.items(): if loggerName.startswith("panda.log"): if len(loggerObj.handlers) == 0: continue diff --git a/pandaharvester/harvestertest/updateEventRangesTest.py b/pandaharvester/harvestertest/updateEventRangesTest.py index 3e4d4f73..a9a831a3 100644 --- a/pandaharvester/harvestertest/updateEventRangesTest.py +++ b/pandaharvester/harvestertest/updateEventRangesTest.py @@ -8,7 +8,7 @@ rID = sys.argv[1] taskid = rID.split("-")[0] -pandaid = long(rID.split("-")[1]) +pandaid = int(rID.split("-")[1]) job = JobSpec() job.PandaID = pandaid diff --git a/pandaharvester/harvestertest/worker_pandajob_dump.py b/pandaharvester/harvestertest/worker_pandajob_dump.py index b52282f1..6c7b1e77 100755 --- a/pandaharvester/harvestertest/worker_pandajob_dump.py +++ b/pandaharvester/harvestertest/worker_pandajob_dump.py @@ -91,7 +91,7 @@ def main(): cursor = conn.cursor() if options.hours: - utcnow = datetime.datetime.utcnow() - datetime.timedelta(hours=options.hours) + utcnow = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) - datetime.timedelta(hours=options.hours) utcnow_str = utcnow.strftime("%Y-%d-%m %H:%M:%S") work_cmd = f'SELECT workerID,batchID,status FROM work_table WHERE modificationTime > "{utcnow_str}"' elif options.workers: diff --git a/pandaharvester/harvesterthrottler/simple_throttler.py b/pandaharvester/harvesterthrottler/simple_throttler.py index 33832975..24e86f2e 100644 --- a/pandaharvester/harvesterthrottler/simple_throttler.py +++ b/pandaharvester/harvesterthrottler/simple_throttler.py @@ -29,7 +29,7 @@ def to_be_throttled(self, queue_config): # loop over all rules criteriaList = [] maxMissedList = [] - timeNow = datetime.datetime.utcnow() + timeNow = core_utils.naive_utcnow() for rule in self.rulesForMissed: # convert rule to criteria if rule["level"] == "site": diff --git a/pandaharvester/harvesterworkermaker/simple_bf_es_worker_maker.py b/pandaharvester/harvesterworkermaker/simple_bf_es_worker_maker.py index fe7155f3..7e7f84cf 100644 --- a/pandaharvester/harvesterworkermaker/simple_bf_es_worker_maker.py +++ b/pandaharvester/harvesterworkermaker/simple_bf_es_worker_maker.py @@ -1,12 +1,8 @@ import datetime import math +import subprocess import traceback -try: - import subprocess32 as subprocess -except Exception: - import subprocess - from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.work_spec import WorkSpec from pandaharvester.harvestermisc.info_utils import PandaQueuesDict @@ -35,7 +31,7 @@ def make_worker(self, jobspec_list, queue_config, job_type, resource_type): tmpLog.debug(f"jobspec_list: {jobspec_list}") workSpec = WorkSpec() - workSpec.creationTime = datetime.datetime.utcnow() + workSpec.creationTime = core_utils.naive_utcnow() # get the queue configuration from the DB panda_queues_dict = PandaQueuesDict() diff --git a/pandaharvester/harvesterworkermaker/simple_worker_maker.py b/pandaharvester/harvesterworkermaker/simple_worker_maker.py index c94d4bff..d25668e5 100644 --- a/pandaharvester/harvesterworkermaker/simple_worker_maker.py +++ b/pandaharvester/harvesterworkermaker/simple_worker_maker.py @@ -1,5 +1,3 @@ -from __future__ import division - import datetime import math import random @@ -88,7 +86,7 @@ def make_worker(self, jobspec_list, queue_config, job_type, resource_type): tmpLog.debug(f"jobspec_list: {jobspec_list}") workSpec = WorkSpec() - workSpec.creationTime = datetime.datetime.utcnow() + workSpec.creationTime = core_utils.naive_utcnow() # get the queue configuration from CRIC panda_queues_dict = PandaQueuesDict() diff --git a/pandaharvester/harvesterzipper/base_zipper.py b/pandaharvester/harvesterzipper/base_zipper.py index c276062a..b047e7de 100644 --- a/pandaharvester/harvesterzipper/base_zipper.py +++ b/pandaharvester/harvesterzipper/base_zipper.py @@ -1,16 +1,12 @@ import gc import multiprocessing import os +import subprocess import tempfile import time import uuid from concurrent.futures import ThreadPoolExecutor as Pool -try: - import subprocess32 as subprocess -except ImportError: - import subprocess - from pandaharvester.harvesterconfig import harvester_config from pandaharvester.harvestercore import core_utils from pandaharvester.harvestercore.plugin_base import PluginBase diff --git a/pandaharvester/panda_pkg_info.py b/pandaharvester/panda_pkg_info.py index cffc32d3..0cda1758 100644 --- a/pandaharvester/panda_pkg_info.py +++ b/pandaharvester/panda_pkg_info.py @@ -1 +1 @@ -release_version = "0.4.2" +release_version = "0.5.0" diff --git a/pyproject.toml b/pyproject.toml index 7663de7e..ece83851 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,19 +17,14 @@ authors = [ dependencies = [ 'requests', 'python-daemon', - 'future', - 'futures; python_version == "2.*"', 'pycryptodomex', 'panda-common', 'pyjwt', - 'subprocess32; python_version == "2.*"', 'rpyc', 'paramiko', 'pexpect', 'psutil >= 5.4.8', - 'scandir; python_version < "3.5"', 'panda-pilot >= 2.7.2.1', - 'six', ] [project.optional-dependencies] diff --git a/setup.py b/setup.py index 7b9e21df..5d929ee6 100644 --- a/setup.py +++ b/setup.py @@ -28,19 +28,14 @@ install_requires=[ "requests", "python-daemon", - "future", - 'futures; python_version == "2.*"', "pycryptodomex", "panda-common", "pyjwt", - 'subprocess32; python_version == "2.*"', "rpyc", "paramiko", "pexpect", "psutil >= 5.4.8", - 'scandir; python_version < "3.5"', "panda-pilot >= 2.7.2.1", - "six", ], # optional pip dependencies extras_require={