From f37099de7ce8a41d1c8f4a93d1772b254428167d Mon Sep 17 00:00:00 2001 From: antoinefalisse Date: Fri, 11 Nov 2022 09:23:19 -0800 Subject: [PATCH 01/41] adjust docker files for running docker locally --- docker/Makefile | 16 ++++------------ docker/docker-compose.yaml | 6 +++--- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index cdf0822b..cc2d5841 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -2,18 +2,10 @@ build: wget -c -O ../mmpose/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth https://mc-opencap-public.s3.us-west-2.amazonaws.com/mmpose_pth/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth wget -c -O ../mmpose/hrnet_w48_coco_wholebody_384x288_dark-f5726563_20200918.pth https://mc-opencap-public.s3.us-west-2.amazonaws.com/mmpose_pth/hrnet_w48_coco_wholebody_384x288_dark-f5726563_20200918.pth - docker build -t 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/opencap .. -f Dockerfile - docker build -t 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/openpose openpose - docker build -t 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/mmpose .. -f mmpose/Dockerfile - -.PHONY: push -push: - aws ecr get-login-password --region us-west-2 --profile opencap | docker login --username AWS --password-stdin 660440363484.dkr.ecr.us-west-2.amazonaws.com - docker push 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/opencap - docker push 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/openpose - docker push 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/mmpose + docker build -t mobilecap .. -f Dockerfile + docker build -t openpose openpose + docker build -t mmpose .. -f mmpose/Dockerfile .PHONY: run run: - aws ecr get-login-password --region us-west-2 --profile opencap | docker login --username AWS --password-stdin 660440363484.dkr.ecr.us-west-2.amazonaws.com - docker-compose up \ No newline at end of file + docker-compose up diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index dce06d31..994f6d29 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,7 +1,7 @@ version: "3.9" services: mobilecap: - image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/opencap + image: opencap volumes: - data:/data env_file: @@ -16,7 +16,7 @@ services: count: 1 capabilities: [gpu] openpose: - image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/openpose + image: openpose volumes: - data:/openpose/data deploy: @@ -27,7 +27,7 @@ services: count: 1 capabilities: [gpu] mmpose: - image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/mmpose + image: mmpose volumes: - data:/mmpose/data deploy: From e7157b067142876ddd5f32d00a4f33c7b96b339f Mon Sep 17 00:00:00 2001 From: antoinefalisse Date: Fri, 11 Nov 2022 12:02:33 -0800 Subject: [PATCH 02/41] openpose bug --- docker/Makefile | 2 +- docker/docker-compose.yaml | 2 +- docker/openpose/Dockerfile | 13 +++++-- docker/openpose/loop.py | 64 +++++++++++++++++--------------- docker/openpose/requirements.txt | 1 + 5 files changed, 47 insertions(+), 35 deletions(-) create mode 100644 docker/openpose/requirements.txt diff --git a/docker/Makefile b/docker/Makefile index cc2d5841..1f707a83 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -2,7 +2,7 @@ build: wget -c -O ../mmpose/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth https://mc-opencap-public.s3.us-west-2.amazonaws.com/mmpose_pth/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth wget -c -O ../mmpose/hrnet_w48_coco_wholebody_384x288_dark-f5726563_20200918.pth https://mc-opencap-public.s3.us-west-2.amazonaws.com/mmpose_pth/hrnet_w48_coco_wholebody_384x288_dark-f5726563_20200918.pth - docker build -t mobilecap .. -f Dockerfile + docker build -t opencap .. -f Dockerfile docker build -t openpose openpose docker build -t mmpose .. -f mmpose/Dockerfile diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 994f6d29..ad999bf8 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,6 +1,6 @@ version: "3.9" services: - mobilecap: + opencap: image: opencap volumes: - data:/data diff --git a/docker/openpose/Dockerfile b/docker/openpose/Dockerfile index 0756e03c..0cb4d962 100644 --- a/docker/openpose/Dockerfile +++ b/docker/openpose/Dockerfile @@ -1,5 +1,10 @@ -RUN pip3 install ffmpeg-python - FROM stanfordnmbl/openpose-gpu:11.3.1 -COPY loop.py /openpose -CMD python loop.py + +WORKDIR /workspace_openpose/ +COPY requirements.txt /workspace_openpose/requirements.txt + +RUN pip3 install --upgrade pip +RUN pip3 install -r requirements.txt + +COPY . /workspace_openpose/ +CMD python3.6 loop.py diff --git a/docker/openpose/loop.py b/docker/openpose/loop.py index b34ef87a..0d0b51a0 100644 --- a/docker/openpose/loop.py +++ b/docker/openpose/loop.py @@ -5,7 +5,7 @@ import ffmpeg #%% -def getVideoRotation(videoPath): +def getVideoOrientation(videoPath): meta = ffmpeg.probe(videoPath) try: @@ -22,8 +22,37 @@ def getVideoRotation(videoPath): rotation = 0 except: rotation = 90 # upright is 90, and intrinsics were captured in that orientation + + if int(rotation) in [0,180]: + horizontal = True + else: + horizontal = False - return int(rotation) + return horizontal + +#%% +def getResolutionCommand(resolutionPoseDetection, horizontal): + + # Adjust OpenPose call based on selected resolution. + if resolutionPoseDetection == 'default': + cmd_hr = ' ' + elif resolutionPoseDetection == '1x1008_4scales': + if horizontal: + cmd_hr = ' --net_resolution "1008x-1" --scale_number 4 --scale_gap 0.25 ' + else: + cmd_hr = ' --net_resolution "-1x1008" --scale_number 4 --scale_gap 0.25 ' + elif resolutionPoseDetection == '1x736': + if horizontal: + cmd_hr = ' --net_resolution "736x-1" ' + else: + cmd_hr = ' --net_resolution "-1x736" ' + elif resolutionPoseDetection == '1x736_2scales': + if horizontal: + cmd_hr = ' --net_resolution "-1x736" --scale_number 2 --scale_gap 0.75 ' + else: + cmd_hr = ' --net_resolution "736x-1" --scale_number 2 --scale_gap 0.75 ' + + return cmd_hr #%% logging.basicConfig(level=logging.INFO) @@ -33,33 +62,8 @@ def getVideoRotation(videoPath): video_path = "/openpose/data/video_openpose.mov" output_dir = "/openpose/data/output_openpose" -rotation = getVideoRotation(video_path) -if rotation in [0,180]: - horizontal = True -else: - horizontal = False - - # Set resolution for OpenPose ('default', '1x736', or '1x1008_4scales'). resolutionPoseDetection = '1x736' -# Adjust OpenPose call based on selected resolution. -if resolutionPoseDetection == 'default': - cmd_hr = ' ' -elif resolutionPoseDetection == '1x1008_4scales': - if horizontal: - cmd_hr = ' --net_resolution "1008x-1" --scale_number 4 --scale_gap 0.25 ' - else: - cmd_hr = ' --net_resolution "-1x1008" --scale_number 4 --scale_gap 0.25 ' -elif resolutionPoseDetection == '1x736': - if horizontal: - cmd_hr = ' --net_resolution "736x-1" ' - else: - cmd_hr = ' --net_resolution "-1x736" ' -elif resolutionPoseDetection == '1x736_2scales': - if horizontal: - cmd_hr = ' --net_resolution "-1x736" --scale_number 2 --scale_gap 0.75 ' - else: - cmd_hr = ' --net_resolution "736x-1" --scale_number 2 --scale_gap 0.75 ' if os.path.isfile(video_path): os.remove(video_path) @@ -74,6 +78,9 @@ def getVideoRotation(videoPath): if os.path.isdir(output_dir): shutil.rmtree(output_dir) os.makedirs(output_dir) + + horizontal = getVideoOrientation(video_path) + cmd_hr = getResolutionCommand(resolutionPoseDetection, horizontal) command = "/openpose/build/examples/openpose/openpose.bin\ --video {video_path}\ @@ -83,5 +90,4 @@ def getVideoRotation(videoPath): os.system(command) logging.info("Done. Cleaning up") - os.remove(video_path) - + os.remove(video_path) \ No newline at end of file diff --git a/docker/openpose/requirements.txt b/docker/openpose/requirements.txt new file mode 100644 index 00000000..33354191 --- /dev/null +++ b/docker/openpose/requirements.txt @@ -0,0 +1 @@ +ffmpeg-python From 572d4375b82a0c7fb9c1dda030524a382b642552 Mon Sep 17 00:00:00 2001 From: antoinefalisse Date: Fri, 11 Nov 2022 13:37:02 -0800 Subject: [PATCH 03/41] fix bugs missing models --- docker/Makefile | 8 ++++---- docker/docker-compose.yaml | 12 ++++++------ docker/openpose/Dockerfile | 5 ++--- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index 1f707a83..aaed245c 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -2,10 +2,10 @@ build: wget -c -O ../mmpose/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth https://mc-opencap-public.s3.us-west-2.amazonaws.com/mmpose_pth/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth wget -c -O ../mmpose/hrnet_w48_coco_wholebody_384x288_dark-f5726563_20200918.pth https://mc-opencap-public.s3.us-west-2.amazonaws.com/mmpose_pth/hrnet_w48_coco_wholebody_384x288_dark-f5726563_20200918.pth - docker build -t opencap .. -f Dockerfile - docker build -t openpose openpose - docker build -t mmpose .. -f mmpose/Dockerfile + docker build -t opencap-dev .. -f Dockerfile + docker build -t openpose-dev openpose --no-cache + docker build -t mmpose-dev .. -f mmpose/Dockerfile .PHONY: run run: - docker-compose up + docker-compose up --remove-orphans diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index ad999bf8..094e695d 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,7 +1,7 @@ version: "3.9" services: - opencap: - image: opencap + opencap-dev: + image: opencap-dev volumes: - data:/data env_file: @@ -15,8 +15,8 @@ services: - driver: nvidia count: 1 capabilities: [gpu] - openpose: - image: openpose + openpose-dev: + image: openpose-dev volumes: - data:/openpose/data deploy: @@ -26,8 +26,8 @@ services: - driver: nvidia count: 1 capabilities: [gpu] - mmpose: - image: mmpose + mmpose-dev: + image: mmpose-dev volumes: - data:/mmpose/data deploy: diff --git a/docker/openpose/Dockerfile b/docker/openpose/Dockerfile index 0cb4d962..eb90e728 100644 --- a/docker/openpose/Dockerfile +++ b/docker/openpose/Dockerfile @@ -1,10 +1,9 @@ FROM stanfordnmbl/openpose-gpu:11.3.1 -WORKDIR /workspace_openpose/ -COPY requirements.txt /workspace_openpose/requirements.txt +COPY requirements.txt /openpose RUN pip3 install --upgrade pip RUN pip3 install -r requirements.txt -COPY . /workspace_openpose/ +COPY loop.py /openpose CMD python3.6 loop.py From 2026c73357b78caa7343c3352eeb9293c6210b5f Mon Sep 17 00:00:00 2001 From: suhlrich <43877159+suhlrich@users.noreply.github.com> Date: Fri, 11 Nov 2022 14:07:51 -0800 Subject: [PATCH 04/41] make sure try statement throws error --- utilsChecker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utilsChecker.py b/utilsChecker.py index d395d522..035b9873 100644 --- a/utilsChecker.py +++ b/utilsChecker.py @@ -299,6 +299,8 @@ def getVideoRotation(videoPath): rotation = 90 else: rotation = 0 + else: + raise Exception('no rotation info') except: rotation = 90 # upright is 90, and intrinsics were captured in that orientation From 282417a91aff2c229258b6c16d81b4f810dca33a Mon Sep 17 00:00:00 2001 From: suhlrich <43877159+suhlrich@users.noreply.github.com> Date: Fri, 11 Nov 2022 14:08:57 -0800 Subject: [PATCH 05/41] fix try except in loop.py --- docker/openpose/loop.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/openpose/loop.py b/docker/openpose/loop.py index 0d0b51a0..c72bb1b2 100644 --- a/docker/openpose/loop.py +++ b/docker/openpose/loop.py @@ -20,6 +20,8 @@ def getVideoOrientation(videoPath): rotation = 90 else: rotation = 0 + else: + raise Exception('no rotation info') except: rotation = 90 # upright is 90, and intrinsics were captured in that orientation @@ -90,4 +92,4 @@ def getResolutionCommand(resolutionPoseDetection, horizontal): os.system(command) logging.info("Done. Cleaning up") - os.remove(video_path) \ No newline at end of file + os.remove(video_path) From 580b705f6a95fb7e2d777d2ce10c1934406bbb9c Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Wed, 8 Feb 2023 16:44:42 -0800 Subject: [PATCH 06/41] remove video if failure --- mmpose/loop_mmpose.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mmpose/loop_mmpose.py b/mmpose/loop_mmpose.py index c9cfb3bb..056b98fb 100644 --- a/mmpose/loop_mmpose.py +++ b/mmpose/loop_mmpose.py @@ -58,3 +58,4 @@ except: logging.info("Pose detection failed.") + os.remove(video_path) From cfa546fcd5fa16a4e089ee38e0eabd55f191bbfe Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Wed, 8 Feb 2023 16:47:45 -0800 Subject: [PATCH 07/41] revert fix --- mmpose/loop_mmpose.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mmpose/loop_mmpose.py b/mmpose/loop_mmpose.py index 056b98fb..c9cfb3bb 100644 --- a/mmpose/loop_mmpose.py +++ b/mmpose/loop_mmpose.py @@ -58,4 +58,3 @@ except: logging.info("Pose detection failed.") - os.remove(video_path) From a90e04a9cd3ad877d5c6d34af8d7f600b6d20669 Mon Sep 17 00:00:00 2001 From: suhlrich Date: Thu, 20 Apr 2023 22:28:30 -0700 Subject: [PATCH 08/41] Update reprocessSessions.py --- Examples/reprocessSessions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Examples/reprocessSessions.py b/Examples/reprocessSessions.py index fb12ea8f..2a185683 100644 --- a/Examples/reprocessSessions.py +++ b/Examples/reprocessSessions.py @@ -55,7 +55,7 @@ # Enter the identifier(s) of the session(s) you want to reprocess. This is a list of one # or more session identifiers. The identifier is found as the 36-character string at the # end of the session url: app.opencap.ai/session/ -session_ids = ['23d52d41-69fe-47cf-8b60-838e4268dd50'] +session_ids = ['a460334c-b2c6-48b4-858d-c998e3cb6d0a'] @@ -68,8 +68,8 @@ # select specific reprocessed. Only one trial (str) is allowed for calib_id and # static_id. A list of strings is allowed for dynamic_ids. -calib_id = [] # None (auto-selected trial), [] (skip), or string of specific trial_id -static_id = [] # None (auto-selected trial), [] (skip), or string of specific trial_id +calib_id = None # None (auto-selected trial), [] (skip), or string of specific trial_id +static_id = None # None (auto-selected trial), [] (skip), or string of specific trial_id dynamic_trialNames = None # None (all dynamic trials), [] (skip), or list of trial names # extract trial ids from trial names From 8ebbd3225a1078e77242e5151ffcae14346eb330 Mon Sep 17 00:00:00 2001 From: suhlrich Date: Thu, 20 Apr 2023 22:59:29 -0700 Subject: [PATCH 09/41] fix --- Examples/reprocessSessions.py | 6 ++++-- utilsServer.py | 10 +++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Examples/reprocessSessions.py b/Examples/reprocessSessions.py index 2a185683..4ff1df1f 100644 --- a/Examples/reprocessSessions.py +++ b/Examples/reprocessSessions.py @@ -68,8 +68,8 @@ # select specific reprocessed. Only one trial (str) is allowed for calib_id and # static_id. A list of strings is allowed for dynamic_ids. -calib_id = None # None (auto-selected trial), [] (skip), or string of specific trial_id -static_id = None # None (auto-selected trial), [] (skip), or string of specific trial_id +calib_id = [] # None (auto-selected trial), [] (skip), or string of specific trial_id +static_id = [] # None (auto-selected trial), [] (skip), or string of specific trial_id dynamic_trialNames = None # None (all dynamic trials), [] (skip), or list of trial names # extract trial ids from trial names @@ -109,3 +109,5 @@ batchReprocess(session_ids,calib_id,static_id,dynamic_ids, resolutionPoseDetection=resolutionPoseDetection, deleteLocalFolder=deleteLocalFolder) + +test = 1 \ No newline at end of file diff --git a/utilsServer.py b/utilsServer.py index 32b44515..0f9aaa70 100644 --- a/utilsServer.py +++ b/utilsServer.py @@ -295,16 +295,16 @@ def batchReprocess(session_ids,calib_id,static_id,dynamic_ids,poseDetector='Open isDocker=isServer, hasWritePermissions = hasWritePermissions) statusData = {'status':'done'} - _ = requests.patch("https://api.opencap.ai/trials/{}/".format(static_id_toProcess), data=statusData, + _ = requests.patch(API_URL + "trials/{}/".format(static_id_toProcess), data=statusData, headers = {"Authorization": "Token {}".format(API_TOKEN)}) except Exception as e: print(e) statusData = {'status':'error'} - _ = requests.patch("https://api.opencap.ai/trials/{}/".format(static_id_toProcess), data=statusData, + _ = requests.patch(API_URL + "trials/{}/".format(static_id_toProcess), data=statusData, headers = {"Authorization": "Token {}".format(API_TOKEN)}) if dynamic_ids == None: - session = requests.get("https://api.opencap.ai/sessions/{}/".format(session_id), + session = requests.get(API_URL + "sessions/{}/".format(session_id), headers = {"Authorization": "Token {}".format(API_TOKEN)}).json() dynamic_ids_toProcess = [t['id'] for t in session['trials'] if (t['name'] != 'calibration' and t['name'] !='neutral')] else: @@ -325,12 +325,12 @@ def batchReprocess(session_ids,calib_id,static_id,dynamic_ids,poseDetector='Open hasWritePermissions = hasWritePermissions) statusData = {'status':'done'} - _ = requests.patch("https://api.opencap.ai/trials/{}/".format(dID), data=statusData, + _ = requests.patch(API_URL + "/trials/{}/".format(dID), data=statusData, headers = {"Authorization": "Token {}".format(API_TOKEN)}) except Exception as e: print(e) statusData = {'status':'error'} - _ = requests.patch("https://api.opencap.ai/trials/{}/".format(static_id_toProcess), data=statusData, + _ = requests.patch(API_URL + "/trials/{}/".format(static_id_toProcess), data=statusData, headers = {"Authorization": "Token {}".format(API_TOKEN)}) def runTestSession(pose='all',isDocker=True): From 12e14c40bbd7690786d711a58128cff649a1c746 Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Tue, 25 Apr 2023 11:44:38 -0700 Subject: [PATCH 10/41] minor cleanup --- Examples/reprocessSessions.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Examples/reprocessSessions.py b/Examples/reprocessSessions.py index 4ff1df1f..fb12ea8f 100644 --- a/Examples/reprocessSessions.py +++ b/Examples/reprocessSessions.py @@ -55,7 +55,7 @@ # Enter the identifier(s) of the session(s) you want to reprocess. This is a list of one # or more session identifiers. The identifier is found as the 36-character string at the # end of the session url: app.opencap.ai/session/ -session_ids = ['a460334c-b2c6-48b4-858d-c998e3cb6d0a'] +session_ids = ['23d52d41-69fe-47cf-8b60-838e4268dd50'] @@ -109,5 +109,3 @@ batchReprocess(session_ids,calib_id,static_id,dynamic_ids, resolutionPoseDetection=resolutionPoseDetection, deleteLocalFolder=deleteLocalFolder) - -test = 1 \ No newline at end of file From 7ee221b012af515522c3c80e2f4ef6eb81a051bf Mon Sep 17 00:00:00 2001 From: suhlrich Date: Wed, 26 Apr 2023 16:35:51 -0700 Subject: [PATCH 11/41] Update requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 40c899a1..589addf8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ opencv-python==4.5.1.48 pyyaml==5.4.1 # tensorflow-gpu==2.4.1 # install with conda -tensorflow-gpu==2.12.0 # enabled for the docker +tensorflow-gpu==2.9.3 # enabled for the docker scipy==1.9.0 pandas==1.1.5 matplotlib==3.3.4 @@ -14,3 +14,4 @@ scikit-learn pingouin==0.5.2 openpyxl ffmpeg-python +psutil \ No newline at end of file From 775fc56516c1acd5c0980f18a15d2192bbd6c272 Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Tue, 29 Aug 2023 09:03:04 -0700 Subject: [PATCH 12/41] minor --- main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 785f06a6..4a9a7b54 100644 --- a/main.py +++ b/main.py @@ -489,12 +489,13 @@ def main(sessionName, trialName, trial_id, camerasToUse=['all'], 'Settings_' + trial_id + '.yaml') settings = { 'poseDetector': poseDetector, - 'resolutionPoseDetection': resolutionPoseDetection, 'augmenter_model': augmenterModel, 'offset': offset, 'imageUpsampleFactor': imageUpsampleFactor, 'openSimModel': sessionMetadata['openSimModel']} - if poseDetector == 'mmpose': + if poseDetector == 'OpenPose': + settings['resolutionPoseDetection'] = resolutionPoseDetection + elif poseDetector == 'mmpose': settings['bbox_thr'] = str(bbox_thr) with open(pathSettings, 'w') as file: yaml.dump(settings, file) From 3e598a76bf505d9d6e27199e4f9dde57c65fa528 Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Tue, 10 Oct 2023 11:37:10 -0700 Subject: [PATCH 13/41] adding cuda check --- app.py | 4 ++-- utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 13a566a5..89b8b253 100644 --- a/app.py +++ b/app.py @@ -10,7 +10,7 @@ import numpy as np from utilsAPI import getAPIURL, getWorkerType from utilsAuth import getToken -from utils import getDataDirectory, checkTime, checkResourceUsage, checkCuda +from utils import getDataDirectory, checkTime, checkResourceUsage, checkCudaTF logging.basicConfig(level=logging.INFO) @@ -59,7 +59,7 @@ logging.info(json.dumps(resourceUsage)) # TODO: not yet functional. # Check GPU device is available - # checkCuda() + checkCudaTF() logging.info(r.text) trial = r.json() diff --git a/utils.py b/utils.py index 70a58a16..70140b63 100644 --- a/utils.py +++ b/utils.py @@ -1480,7 +1480,7 @@ def checkResourceUsage(stop_machine_and_email=True): return resourceUsage -def checkCuda(): +def checkCudaTF(): import tensorflow as tf if tf.config.experimental.list_physical_devices('GPU'): From a765f0e6c5034056fabc193e50eab2f785fe7edb Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Tue, 10 Oct 2023 11:39:51 -0700 Subject: [PATCH 14/41] adding cuda check --- app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app.py b/app.py index 89b8b253..ee28517e 100644 --- a/app.py +++ b/app.py @@ -59,7 +59,9 @@ logging.info(json.dumps(resourceUsage)) # TODO: not yet functional. # Check GPU device is available + logging.info('Checking GPU device is available...') checkCudaTF() + logging.info('GPU device is available.') logging.info(r.text) trial = r.json() From 316b96aab0b10ec6b16c7cd4d5a93ff3ec10b369 Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Tue, 10 Oct 2023 11:45:44 -0700 Subject: [PATCH 15/41] adding cuda check --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 70140b63..4afdd492 100644 --- a/utils.py +++ b/utils.py @@ -1489,7 +1489,7 @@ def checkCudaTF(): print(f"Found {len(gpus)} GPU(s).") # You can also print GPU device names and memory limits if needed for gpu in gpus: - print(f"GPU: {gpu.name}, Memory: {gpu.memory_limit}") + print(f"GPU: {gpu.name}") else: message = "Cuda check failed on an OpenCap machine backend machine: " \ + socket.gethostname() + ". It has been stopped." From 6853d3e6c0c4abbdb7402a0ca1debb6742b9f96e Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Tue, 10 Oct 2023 11:48:29 -0700 Subject: [PATCH 16/41] adding cuda check --- utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils.py b/utils.py index 4afdd492..00301915 100644 --- a/utils.py +++ b/utils.py @@ -1485,6 +1485,8 @@ def checkCudaTF(): if tf.config.experimental.list_physical_devices('GPU'): # Get the list of available GPUs + + print("HERE") gpus = tf.config.experimental.list_physical_devices('GPU') print(f"Found {len(gpus)} GPU(s).") # You can also print GPU device names and memory limits if needed From 01344041a76b4a473d9dc86c1f28d0c1ffba90e9 Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Tue, 10 Oct 2023 11:52:42 -0700 Subject: [PATCH 17/41] adding cuda check --- utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils.py b/utils.py index 00301915..9b6d0ce3 100644 --- a/utils.py +++ b/utils.py @@ -1487,11 +1487,11 @@ def checkCudaTF(): # Get the list of available GPUs print("HERE") - gpus = tf.config.experimental.list_physical_devices('GPU') + gpus = tf.config.list_physical_devices('GPU') print(f"Found {len(gpus)} GPU(s).") # You can also print GPU device names and memory limits if needed - for gpu in gpus: - print(f"GPU: {gpu.name}") + # for gpu in gpus: + # print(f"GPU: {gpu.name}") else: message = "Cuda check failed on an OpenCap machine backend machine: " \ + socket.gethostname() + ". It has been stopped." From 339975b1bfc1cf9d1bcdedf6f653dc87f681032d Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Tue, 10 Oct 2023 11:56:14 -0700 Subject: [PATCH 18/41] adding cuda check --- app.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index ee28517e..7e6755af 100644 --- a/app.py +++ b/app.py @@ -11,6 +11,7 @@ from utilsAPI import getAPIURL, getWorkerType from utilsAuth import getToken from utils import getDataDirectory, checkTime, checkResourceUsage, checkCudaTF +import tensorflow as tf logging.basicConfig(level=logging.INFO) @@ -60,7 +61,12 @@ # TODO: not yet functional. # Check GPU device is available logging.info('Checking GPU device is available...') - checkCudaTF() + # checkCudaTF() + + print("HERE") + gpus = tf.config.list_physical_devices('GPU') + print(f"Found {len(gpus)} GPU(s).") + logging.info('GPU device is available.') logging.info(r.text) From 356e12389a473b0dcefdf9efc98e8da095f2cb61 Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Tue, 10 Oct 2023 12:04:43 -0700 Subject: [PATCH 19/41] testing cuda --- app.py | 10 ++++++---- utilsAugmenter.py | 5 +++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index 7e6755af..ba4942d5 100644 --- a/app.py +++ b/app.py @@ -63,11 +63,13 @@ logging.info('Checking GPU device is available...') # checkCudaTF() - print("HERE") - gpus = tf.config.list_physical_devices('GPU') - print(f"Found {len(gpus)} GPU(s).") + visible_devices = tf.config.list_physical_devices() + logging.info("Visible devices app.py:", visible_devices) - logging.info('GPU device is available.') + # gpus = tf.config.list_physical_devices('GPU') + # print(f"Found {len(gpus)} GPU(s).") + + # logging.info('GPU device is available.') logging.info(r.text) trial = r.json() diff --git a/utilsAugmenter.py b/utilsAugmenter.py index f10c4b1c..ce519039 100644 --- a/utilsAugmenter.py +++ b/utilsAugmenter.py @@ -5,6 +5,7 @@ import tensorflow as tf from utils import TRC2numpy import json +import logging def augmentTRC(pathInputTRCFile, subject_mass, subject_height, pathOutputTRCFile, augmenterDir, augmenterModelName="LSTM", @@ -110,6 +111,10 @@ def augmentTRC(pathInputTRCFile, subject_mass, subject_height, json_file = open(os.path.join(augmenterModelDir, "model.json"), 'r') pretrainedModel_json = json_file.read() json_file.close() + + visible_devices = tf.config.list_physical_devices() + logging.info("Visible devices - augmentation:", visible_devices) + model = tf.keras.models.model_from_json(pretrainedModel_json) model.load_weights(os.path.join(augmenterModelDir, "weights.h5")) outputs = model.predict(inputs) From ee6f5555943b5272a869961dd58547cfea19729c Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Wed, 11 Oct 2023 14:52:38 -0700 Subject: [PATCH 20/41] testing cuda check --- app.py | 14 ++------------ openpose/loop_openpose.py | 16 +++++++++++++++- utils.py | 10 +++------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/app.py b/app.py index ba4942d5..13a566a5 100644 --- a/app.py +++ b/app.py @@ -10,8 +10,7 @@ import numpy as np from utilsAPI import getAPIURL, getWorkerType from utilsAuth import getToken -from utils import getDataDirectory, checkTime, checkResourceUsage, checkCudaTF -import tensorflow as tf +from utils import getDataDirectory, checkTime, checkResourceUsage, checkCuda logging.basicConfig(level=logging.INFO) @@ -60,16 +59,7 @@ logging.info(json.dumps(resourceUsage)) # TODO: not yet functional. # Check GPU device is available - logging.info('Checking GPU device is available...') - # checkCudaTF() - - visible_devices = tf.config.list_physical_devices() - logging.info("Visible devices app.py:", visible_devices) - - # gpus = tf.config.list_physical_devices('GPU') - # print(f"Found {len(gpus)} GPU(s).") - - # logging.info('GPU device is available.') + # checkCuda() logging.info(r.text) trial = r.json() diff --git a/openpose/loop_openpose.py b/openpose/loop_openpose.py index 617b9d3a..033839b2 100644 --- a/openpose/loop_openpose.py +++ b/openpose/loop_openpose.py @@ -4,8 +4,21 @@ import shutil import ffmpeg import json +import subprocess -#%% +#%% +def check_cuda_device(): + try: + # Run the nvidia-smi command and capture the output + _ = subprocess.check_output(["nvidia-smi"]) + # If the command ran successfully, assume a CUDA device is present + print("A CUDA-capable device is detected.") + except subprocess.CalledProcessError as e: + # If the command fails, it means no CUDA device is detected + print("No CUDA-capable device is detected. Error:", e) + raise Exception("No CUDA-capable device is detected.") + +#%% def getVideoOrientation(videoPath): meta = ffmpeg.probe(videoPath) @@ -87,6 +100,7 @@ def getResolutionCommand(resolutionPoseDetection, horizontal): horizontal = getVideoOrientation(video_path) cmd_hr = getResolutionCommand(resolutionPoseDetection, horizontal) + check_cuda_device() command = "/openpose/build/examples/openpose/openpose.bin\ --video {video_path}\ --display 0\ diff --git a/utils.py b/utils.py index 9b6d0ce3..2557c45e 100644 --- a/utils.py +++ b/utils.py @@ -1483,15 +1483,11 @@ def checkResourceUsage(stop_machine_and_email=True): def checkCudaTF(): import tensorflow as tf - if tf.config.experimental.list_physical_devices('GPU'): - # Get the list of available GPUs - - print("HERE") + if tf.config.list_physical_devices('GPU'): gpus = tf.config.list_physical_devices('GPU') print(f"Found {len(gpus)} GPU(s).") - # You can also print GPU device names and memory limits if needed - # for gpu in gpus: - # print(f"GPU: {gpu.name}") + for gpu in gpus: + print(f"GPU: {gpu.name}") else: message = "Cuda check failed on an OpenCap machine backend machine: " \ + socket.gethostname() + ". It has been stopped." From 211cdaa5070fe5535aa663795ac509f2a42d9a57 Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Wed, 11 Oct 2023 14:53:23 -0700 Subject: [PATCH 21/41] minor --- openpose/loop_openpose.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openpose/loop_openpose.py b/openpose/loop_openpose.py index 033839b2..82850aca 100644 --- a/openpose/loop_openpose.py +++ b/openpose/loop_openpose.py @@ -6,16 +6,18 @@ import json import subprocess +logging.basicConfig(level=logging.INFO) + #%% def check_cuda_device(): try: # Run the nvidia-smi command and capture the output _ = subprocess.check_output(["nvidia-smi"]) # If the command ran successfully, assume a CUDA device is present - print("A CUDA-capable device is detected.") + logging.info("A CUDA-capable device is detected.") except subprocess.CalledProcessError as e: # If the command fails, it means no CUDA device is detected - print("No CUDA-capable device is detected. Error:", e) + logging.info("No CUDA-capable device is detected. Error:", e) raise Exception("No CUDA-capable device is detected.") #%% @@ -71,7 +73,6 @@ def getResolutionCommand(resolutionPoseDetection, horizontal): return cmd_hr #%% -logging.basicConfig(level=logging.INFO) logging.info("Waiting for data...") From 4160fabd9b2324e837186b25d6050707a7ccc611 Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Wed, 11 Oct 2023 14:55:51 -0700 Subject: [PATCH 22/41] minor --- app.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app.py b/app.py index 13a566a5..d5d4c492 100644 --- a/app.py +++ b/app.py @@ -10,7 +10,7 @@ import numpy as np from utilsAPI import getAPIURL, getWorkerType from utilsAuth import getToken -from utils import getDataDirectory, checkTime, checkResourceUsage, checkCuda +from utils import getDataDirectory, checkTime, checkResourceUsage logging.basicConfig(level=logging.INFO) @@ -57,9 +57,6 @@ # Check resource usage resourceUsage = checkResourceUsage(stop_machine_and_email=True) logging.info(json.dumps(resourceUsage)) - # TODO: not yet functional. - # Check GPU device is available - # checkCuda() logging.info(r.text) trial = r.json() From 57b26d01bae04cdba3fe86879d5cea1a9a1bb52a Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Wed, 11 Oct 2023 14:58:26 -0700 Subject: [PATCH 23/41] test crash --- openpose/loop_openpose.py | 1 + utilsAugmenter.py | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/openpose/loop_openpose.py b/openpose/loop_openpose.py index 82850aca..43f71e35 100644 --- a/openpose/loop_openpose.py +++ b/openpose/loop_openpose.py @@ -15,6 +15,7 @@ def check_cuda_device(): _ = subprocess.check_output(["nvidia-smi"]) # If the command ran successfully, assume a CUDA device is present logging.info("A CUDA-capable device is detected.") + raise Exception("No CUDA-capable device is detected.") except subprocess.CalledProcessError as e: # If the command fails, it means no CUDA device is detected logging.info("No CUDA-capable device is detected. Error:", e) diff --git a/utilsAugmenter.py b/utilsAugmenter.py index ce519039..f10c4b1c 100644 --- a/utilsAugmenter.py +++ b/utilsAugmenter.py @@ -5,7 +5,6 @@ import tensorflow as tf from utils import TRC2numpy import json -import logging def augmentTRC(pathInputTRCFile, subject_mass, subject_height, pathOutputTRCFile, augmenterDir, augmenterModelName="LSTM", @@ -111,10 +110,6 @@ def augmentTRC(pathInputTRCFile, subject_mass, subject_height, json_file = open(os.path.join(augmenterModelDir, "model.json"), 'r') pretrainedModel_json = json_file.read() json_file.close() - - visible_devices = tf.config.list_physical_devices() - logging.info("Visible devices - augmentation:", visible_devices) - model = tf.keras.models.model_from_json(pretrainedModel_json) model.load_weights(os.path.join(augmenterModelDir, "weights.h5")) outputs = model.predict(inputs) From f27bdc1867fdc5d3e894dce0ea967334a86628cc Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Wed, 11 Oct 2023 15:11:43 -0700 Subject: [PATCH 24/41] testing both openpose and mmpose --- mmpose/loop_mmpose.py | 10 ++++++++++ openpose/loop_openpose.py | 1 - utilsServer.py | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/mmpose/loop_mmpose.py b/mmpose/loop_mmpose.py index d5ebd116..4bedc363 100644 --- a/mmpose/loop_mmpose.py +++ b/mmpose/loop_mmpose.py @@ -3,6 +3,7 @@ import logging import shutil import json +import torch from utilsMMpose import detection_inference, pose_inference @@ -10,6 +11,14 @@ logging.info("Waiting for data...") +def checkCudaPyTorch(): + if torch.cuda.is_available(): + num_gpus = torch.cuda.device_count() + logging.info(f"Found {num_gpus} GPU(s).") + else: + logging.info("No GPU detected. Exiting.") + raise Exception("No GPU detected. Exiting.") + video_path = "/mmpose/data/video_mmpose.mov" output_dir = "/mmpose/data/output_mmpose" @@ -38,6 +47,7 @@ os.makedirs(output_dir) try: + checkCudaPyTorch() # Run human detection. pathModelCkptPerson = model_ckpt_person bboxPath = os.path.join(output_dir, 'box.pkl') diff --git a/openpose/loop_openpose.py b/openpose/loop_openpose.py index 43f71e35..82850aca 100644 --- a/openpose/loop_openpose.py +++ b/openpose/loop_openpose.py @@ -15,7 +15,6 @@ def check_cuda_device(): _ = subprocess.check_output(["nvidia-smi"]) # If the command ran successfully, assume a CUDA device is present logging.info("A CUDA-capable device is detected.") - raise Exception("No CUDA-capable device is detected.") except subprocess.CalledProcessError as e: # If the command fails, it means no CUDA device is detected logging.info("No CUDA-capable device is detected. Error:", e) diff --git a/utilsServer.py b/utilsServer.py index 73b9d2df..34f5acb0 100644 --- a/utilsServer.py +++ b/utilsServer.py @@ -443,6 +443,7 @@ def runTestSession(pose='all',isDocker=True): # trials['failure'] = '698162c8-3980-46e5-a3c5-8d4f081db4c4' # failed trial for testing else: # dev trials trials['openpose'] = '89d77579-8371-4760-a019-95f2c793622c' # session acd0e19c-6c86-4ba4-95fd-94b97229a926 + trials['hrnet'] = 'e0e02393-42ee-46d4-9ae1-a6fbb0b89c42' # session 3510c726-a1b8-4de4-a4a2-52b021b4aab2 if pose == 'all': trialList = list(trials.values()) From a3682ba5e432c33468fa3e96b6b700f8180784ab Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Fri, 16 Feb 2024 10:29:03 -0800 Subject: [PATCH 25/41] test --- utilsChecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilsChecker.py b/utilsChecker.py index 0ccea82b..42fe4204 100644 --- a/utilsChecker.py +++ b/utilsChecker.py @@ -1879,7 +1879,7 @@ def filterKeypointsButterworth(key2D,filtFreq,sampleFreq,order=4): wn=0.99 elif wn==1: wn=0.99 - + print("Filter frequency: " + str(filtFreq) + " Hz") sos = butter(order/2,wn,btype='low',output='sos') for i in range(2): From cf31ea1074b30a6144ad705bf99dbbe1b8dd41a7 Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Fri, 16 Feb 2024 10:50:47 -0800 Subject: [PATCH 26/41] roll back --- utilsChecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilsChecker.py b/utilsChecker.py index 42fe4204..0ccea82b 100644 --- a/utilsChecker.py +++ b/utilsChecker.py @@ -1879,7 +1879,7 @@ def filterKeypointsButterworth(key2D,filtFreq,sampleFreq,order=4): wn=0.99 elif wn==1: wn=0.99 - print("Filter frequency: " + str(filtFreq) + " Hz") + sos = butter(order/2,wn,btype='low',output='sos') for i in range(2): From 8d9eb18f32004466bed7b91703f802d1e9fa65be Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Mon, 17 Jun 2024 16:55:50 -0700 Subject: [PATCH 27/41] push images built locally to ECR --- docker/Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker/Makefile b/docker/Makefile index f598e1d9..aa373dae 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -6,6 +6,14 @@ build: docker build -t openpose-dev .. -f openpose/Dockerfile docker build -t mmpose-dev .. -f mmpose/Dockerfile +# Push the image to ECR after building dev locally +.PHONY: push +push: + aws ecr get-login-password --region us-west-2 --profile opencap | docker login --username AWS --password-stdin 660440363484.dkr.ecr.us-west-2.amazonaws.com + docker push 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/opencap-dev + docker push 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/openpose-dev + docker push 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/mmpose-dev + .PHONY: run run: docker-compose up --remove-orphans From 97e2a4af0e5909649a58540992db49fbebdd47ff Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Tue, 18 Jun 2024 10:35:50 -0700 Subject: [PATCH 28/41] roll back makefile changes --- docker/Makefile | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index aa373dae..f598e1d9 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -6,14 +6,6 @@ build: docker build -t openpose-dev .. -f openpose/Dockerfile docker build -t mmpose-dev .. -f mmpose/Dockerfile -# Push the image to ECR after building dev locally -.PHONY: push -push: - aws ecr get-login-password --region us-west-2 --profile opencap | docker login --username AWS --password-stdin 660440363484.dkr.ecr.us-west-2.amazonaws.com - docker push 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/opencap-dev - docker push 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/openpose-dev - docker push 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/mmpose-dev - .PHONY: run run: docker-compose up --remove-orphans From 80873ccc0ac01e9ed30f0b0f57b00900942d1b14 Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Thu, 27 Jun 2024 12:06:00 -0700 Subject: [PATCH 29/41] minor --- app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 9852e267..8412343e 100644 --- a/app.py +++ b/app.py @@ -122,11 +122,12 @@ continue # This is a hack to have the trials with status "reprocess" to be reprocessed - # with camerasToUse_c = ['all_available'] instead of ['all']. In practice, this + # with camerasToUse = ['all_available'] instead of ['all']. In practice, this # allows reprocessing on server trials that failed because video(s) were not available. # This is a temporary solution until we have a better way to handle this. By default, # trials with missing videos are error-ed out directly so that we do not spend time - # on processing them. + # on processing them. Problem is that these trials then don't have pose pickles, which + # makes is hard for local reprocessing (hrnet still a hassle to install locally on Windows). status = trial["status"] if status == "reprocess": camerasToUse_c = ['all_available'] From 71fa08753f8710cea779fdfe083e32396b8e5b44 Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Fri, 5 Jul 2024 12:11:20 -0700 Subject: [PATCH 30/41] test --- utilsServer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utilsServer.py b/utilsServer.py index f97877c5..3bbe7aae 100644 --- a/utilsServer.py +++ b/utilsServer.py @@ -464,6 +464,7 @@ def runTestSession(pose='all',isDocker=True): for trial_id in trialList: trial = getTrialJson(trial_id) logging.info("Running status check on trial name: " + trial['name'] + "_" + str(trial_id) + "\n\n") + raise ValueError('test') processTrial(trial["session"], trial_id, trial_type='static', isDocker=isDocker) except: logging.info("test trial failed. stopping machine.") From b0e2027761aa5a16b5f33d7673acc57d63bbc06e Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Fri, 5 Jul 2024 12:19:33 -0700 Subject: [PATCH 31/41] test --- utilsAPI.py | 4 ++++ utilsServer.py | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/utilsAPI.py b/utilsAPI.py index 3f83c514..99904ae7 100644 --- a/utilsAPI.py +++ b/utilsAPI.py @@ -42,6 +42,10 @@ def getStatusEmails(): emailInfo['ip'] = json.loads(config("STATUS_EMAIL_IP")) except: emailInfo = None + try: + emailInfo['ip'] = json.loads(config("STATUS_EMAIL_IP")) + except: + pass return emailInfo diff --git a/utilsServer.py b/utilsServer.py index 3bbe7aae..f97877c5 100644 --- a/utilsServer.py +++ b/utilsServer.py @@ -464,7 +464,6 @@ def runTestSession(pose='all',isDocker=True): for trial_id in trialList: trial = getTrialJson(trial_id) logging.info("Running status check on trial name: " + trial['name'] + "_" + str(trial_id) + "\n\n") - raise ValueError('test') processTrial(trial["session"], trial_id, trial_type='static', isDocker=isDocker) except: logging.info("test trial failed. stopping machine.") From 63d7c3930d57662698e4ad72d268a570271b3847 Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Fri, 5 Jul 2024 12:27:01 -0700 Subject: [PATCH 32/41] testing --- utilsServer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utilsServer.py b/utilsServer.py index f97877c5..b80164d6 100644 --- a/utilsServer.py +++ b/utilsServer.py @@ -464,6 +464,7 @@ def runTestSession(pose='all',isDocker=True): for trial_id in trialList: trial = getTrialJson(trial_id) logging.info("Running status check on trial name: " + trial['name'] + "_" + str(trial_id) + "\n\n") + raise Exception('Simulated error') processTrial(trial["session"], trial_id, trial_type='static', isDocker=isDocker) except: logging.info("test trial failed. stopping machine.") From 0e04183f0f4da07ae5657f43fc2d7f50eff7b8e2 Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Fri, 5 Jul 2024 12:31:13 -0700 Subject: [PATCH 33/41] minor --- utilsAPI.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utilsAPI.py b/utilsAPI.py index 99904ae7..982bf4f7 100644 --- a/utilsAPI.py +++ b/utilsAPI.py @@ -39,11 +39,10 @@ def getStatusEmails(): emailInfo['fromEmail'] = config("STATUS_EMAIL_FROM") emailInfo['password'] = config("STATUS_EMAIL_FROM_PW") emailInfo['toEmails'] = json.loads(config("STATUS_EMAIL_TO")) - emailInfo['ip'] = json.loads(config("STATUS_EMAIL_IP")) except: emailInfo = None try: - emailInfo['ip'] = json.loads(config("STATUS_EMAIL_IP")) + emailInfo['ip'] = config("STATUS_EMAIL_IP") except: pass From f659fc889d7ff4c10a555eedc38ecbb7a5840a89 Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Fri, 5 Jul 2024 12:33:08 -0700 Subject: [PATCH 34/41] minor --- utilsServer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utilsServer.py b/utilsServer.py index b80164d6..f97877c5 100644 --- a/utilsServer.py +++ b/utilsServer.py @@ -464,7 +464,6 @@ def runTestSession(pose='all',isDocker=True): for trial_id in trialList: trial = getTrialJson(trial_id) logging.info("Running status check on trial name: " + trial['name'] + "_" + str(trial_id) + "\n\n") - raise Exception('Simulated error') processTrial(trial["session"], trial_id, trial_type='static', isDocker=isDocker) except: logging.info("test trial failed. stopping machine.") From 0e875e116b385969ed2451a8a3a5f66f7260ee8d Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Mon, 30 Sep 2024 15:08:02 +0200 Subject: [PATCH 35/41] docker-compose should use dev images --- docker/docker-compose.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index dce06d31..87223113 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,7 +1,7 @@ version: "3.9" services: mobilecap: - image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/opencap + image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/opencap-dev volumes: - data:/data env_file: @@ -16,7 +16,7 @@ services: count: 1 capabilities: [gpu] openpose: - image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/openpose + image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/openpose-dev volumes: - data:/openpose/data deploy: @@ -27,7 +27,7 @@ services: count: 1 capabilities: [gpu] mmpose: - image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/mmpose + image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/mmpose-dev volumes: - data:/mmpose/data deploy: From 15aea14f9f2790515ed54475d964ad2da6496ed1 Mon Sep 17 00:00:00 2001 From: antoinefalisse Date: Mon, 30 Sep 2024 17:31:25 -0700 Subject: [PATCH 36/41] get local client info: git commit and hostname --- app.py | 6 ++++-- docker/Dockerfile | 3 +++ docker/Makefile | 4 +++- utils.py | 20 +++++++++++++++++++- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index 9f5c131e..f838c199 100644 --- a/app.py +++ b/app.py @@ -11,7 +11,8 @@ from utilsAPI import getAPIURL, getWorkerType, getASInstance, unprotect_current_instance, get_number_of_pending_trials from utilsAuth import getToken from utils import (getDataDirectory, checkTime, checkResourceUsage, - sendStatusEmail, checkForTrialsWithStatus) + sendStatusEmail, checkForTrialsWithStatus, + getCommitHash, getHostname, postLocalClientInfo) logging.basicConfig(level=logging.INFO) @@ -72,7 +73,7 @@ continue if r.status_code == 404: - logging.info("...pulling " + workerType + " trials.") + logging.info("...pulling " + workerType + " trials from " + API_URL) time.sleep(1) # When using autoscaling, we will remove the instance scale-in protection if it hasn't @@ -137,6 +138,7 @@ logging.info("processTrial({},{},trial_type={})".format(trial["session"], trial["id"], trial_type)) try: + postLocalClientInfo(trial_url) # trigger reset of timer for last processed trial processTrial(trial["session"], trial["id"], trial_type=trial_type, isDocker=isDocker) # note a result needs to be posted for the API to know we finished, but we are posting them diff --git a/docker/Dockerfile b/docker/Dockerfile index a7494c0e..04ea4a9d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,4 +15,7 @@ RUN pip3 install -r requirements.txt COPY . /workspace/ +ARG GIT_COMMIT_HASH +ENV GIT_COMMIT_HASH=${GIT_COMMIT_HASH} + CMD python3.8 app.py diff --git a/docker/Makefile b/docker/Makefile index f598e1d9..84caa0aa 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,8 +1,10 @@ +GIT_COMMIT_HASH=$(shell git rev-parse --short HEAD) + .PHONY: build build: wget -c -O ../mmpose/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth https://mc-opencap-public.s3.us-west-2.amazonaws.com/mmpose_pth/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth wget -c -O ../mmpose/hrnet_w48_coco_wholebody_384x288_dark-f5726563_20200918.pth https://mc-opencap-public.s3.us-west-2.amazonaws.com/mmpose_pth/hrnet_w48_coco_wholebody_384x288_dark-f5726563_20200918.pth - docker build -t opencap-dev .. -f Dockerfile + docker build --build-arg GIT_COMMIT_HASH=$(GIT_COMMIT_HASH) -t opencap-dev .. -f Dockerfile docker build -t openpose-dev .. -f openpose/Dockerfile docker build -t mmpose-dev .. -f mmpose/Dockerfile diff --git a/utils.py b/utils.py index f763f4ca..e083d449 100644 --- a/utils.py +++ b/utils.py @@ -1612,4 +1612,22 @@ def get_entry_with_largest_number(trialList): except ValueError: continue - return max_entry \ No newline at end of file + return max_entry + +# Get local client info and update + +def getCommitHash(): + return os.getenv('GIT_COMMIT_HASH') + +def getHostname(): + return socket.gethostname() + +def postLocalClientInfo(trial_url): + data = { + "git_commit": getCommitHash(), + "hostname": getHostname() + } + r = requests.patch(trial_url, data=data, + headers = {"Authorization": "Token {}".format(API_TOKEN)}) + + return r From b5c0b03e547e900b71a7e1a6908f12982932c059 Mon Sep 17 00:00:00 2001 From: antoinefalisse Date: Thu, 3 Oct 2024 15:49:54 -0700 Subject: [PATCH 37/41] add info for processed duration --- app.py | 13 ++++++++++--- utils.py | 9 +++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index f838c199..b56719a9 100644 --- a/app.py +++ b/app.py @@ -7,12 +7,14 @@ import traceback import logging import glob +from datetime import datetime, timedelta import numpy as np from utilsAPI import getAPIURL, getWorkerType, getASInstance, unprotect_current_instance, get_number_of_pending_trials from utilsAuth import getToken from utils import (getDataDirectory, checkTime, checkResourceUsage, sendStatusEmail, checkForTrialsWithStatus, - getCommitHash, getHostname, postLocalClientInfo) + getCommitHash, getHostname, postLocalClientInfo, + postProcessedDuration) logging.basicConfig(level=logging.INFO) @@ -139,8 +141,13 @@ try: postLocalClientInfo(trial_url) - # trigger reset of timer for last processed trial - processTrial(trial["session"], trial["id"], trial_type=trial_type, isDocker=isDocker) + process_start_time = datetime.now() + + # trigger reset of timer for last processed trial + processTrial(trial["session"], trial["id"], trial_type=trial_type, isDocker=isDocker) + process_end_time = datetime.now() + postProcessedDuration(trial_url, process_end_time - process_start_time) + # note a result needs to be posted for the API to know we finished, but we are posting them # automatically thru procesTrial now r = requests.patch(trial_url, data={"status": "done"}, diff --git a/utils.py b/utils.py index e083d449..5e071482 100644 --- a/utils.py +++ b/utils.py @@ -1631,3 +1631,12 @@ def postLocalClientInfo(trial_url): headers = {"Authorization": "Token {}".format(API_TOKEN)}) return r + +def postProcessedDuration(trial_url, duration): + data = { + "processed_duration": duration + } + r = requests.patch(trial_url, data=data, + headers = {"Authorization": "Token {}".format(API_TOKEN)}) + + return r From 878b32e67eced6193f6dd51a7a2b682149dae5f8 Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Thu, 18 Jul 2024 10:56:07 -0700 Subject: [PATCH 38/41] gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d5e8ba79..e94340b0 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ Examples/Data/* # Files for re-processing Examples/reprocessSessionServer.py +Examples/reprocessDataServer.py *.ini *.stats From dd35c47f6751d294f9bbe147f218ddb8db69f8da Mon Sep 17 00:00:00 2001 From: Antoine Falisse Date: Mon, 30 Sep 2024 15:08:02 +0200 Subject: [PATCH 39/41] docker-compose should use dev images --- docker/docker-compose.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 094e695d..87223113 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,7 +1,7 @@ version: "3.9" services: - opencap-dev: - image: opencap-dev + mobilecap: + image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/opencap-dev volumes: - data:/data env_file: @@ -15,8 +15,8 @@ services: - driver: nvidia count: 1 capabilities: [gpu] - openpose-dev: - image: openpose-dev + openpose: + image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/openpose-dev volumes: - data:/openpose/data deploy: @@ -26,8 +26,8 @@ services: - driver: nvidia count: 1 capabilities: [gpu] - mmpose-dev: - image: mmpose-dev + mmpose: + image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/mmpose-dev volumes: - data:/mmpose/data deploy: From 90112bb2a0cc7e95f0b87e36bc055dbde96444d7 Mon Sep 17 00:00:00 2001 From: clarkadmin Date: Mon, 7 Oct 2024 18:35:01 -0700 Subject: [PATCH 40/41] change makefile back to dev version and add commit hash --- docker/Makefile | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index 84caa0aa..b2709708 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,13 +1,42 @@ +BASE_NAME := 660440363484.dkr.ecr.us-west-2.amazonaws.com +REPO_NAME := opencap +PROD_BRANCH := main + +# Determine the branch name +CURRENT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) + +# Determine image tag based on branch +ifeq ($(CURRENT_BRANCH),$(PROD_BRANCH)) + OPENCAP_IMAGE_TAG := opencap + OPENPOSE_IMAGE_TAG := openpose + MMPOSE_IMAGE_TAG := mmpose +else + OPENCAP_IMAGE_TAG := opencap-dev + OPENPOSE_IMAGE_TAG := openpose-dev + MMPOSE_IMAGE_TAG := mmpose-dev +endif + +# Get git commit hash info to pass it into container GIT_COMMIT_HASH=$(shell git rev-parse --short HEAD) .PHONY: build build: wget -c -O ../mmpose/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth https://mc-opencap-public.s3.us-west-2.amazonaws.com/mmpose_pth/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth wget -c -O ../mmpose/hrnet_w48_coco_wholebody_384x288_dark-f5726563_20200918.pth https://mc-opencap-public.s3.us-west-2.amazonaws.com/mmpose_pth/hrnet_w48_coco_wholebody_384x288_dark-f5726563_20200918.pth - docker build --build-arg GIT_COMMIT_HASH=$(GIT_COMMIT_HASH) -t opencap-dev .. -f Dockerfile - docker build -t openpose-dev .. -f openpose/Dockerfile - docker build -t mmpose-dev .. -f mmpose/Dockerfile + + docker build --build-arg GIT_COMMIT_HASH=$(GIT_COMMIT_HASH) -t $(BASE_NAME)/$(REPO_NAME)/$(OPENCAP_IMAGE_TAG) .. -f Dockerfile + docker build -t $(BASE_NAME)/$(REPO_NAME)/$(OPENPOSE_IMAGE_TAG) .. -f openpose/Dockerfile + docker build -t $(BASE_NAME)/$(REPO_NAME)/$(MMPOSE_IMAGE_TAG) .. -f mmpose/Dockerfile + +.PHONY: push +push: + aws ecr get-login-password --region us-west-2 --profile opencap | docker login --username AWS --password-stdin 660440363484.dkr.ecr.us-west-2.amazonaws.com + + docker push $(BASE_NAME)/$(REPO_NAME)/$(OPENCAP_IMAGE_TAG) + docker push $(BASE_NAME)/$(REPO_NAME)/$(OPENPOSE_IMAGE_TAG) + docker push $(BASE_NAME)/$(REPO_NAME)/$(MMPOSE_IMAGE_TAG) .PHONY: run run: - docker-compose up --remove-orphans + aws ecr get-login-password --region us-west-2 --profile opencap | docker login --username AWS --password-stdin 660440363484.dkr.ecr.us-west-2.amazonaws.com + docker-compose up \ No newline at end of file From 21ec161a35fe310134748c644c327f73fa804dbc Mon Sep 17 00:00:00 2001 From: clarkadmin Date: Tue, 8 Oct 2024 11:05:44 -0700 Subject: [PATCH 41/41] address comments and add more documentation --- app.py | 3 +++ docker/docker-compose.yaml | 6 +++--- utils.py | 11 +++++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index b56719a9..e82ab7f3 100644 --- a/app.py +++ b/app.py @@ -140,11 +140,14 @@ logging.info("processTrial({},{},trial_type={})".format(trial["session"], trial["id"], trial_type)) try: + # Post new client info to Trial and start timer for processing duration postLocalClientInfo(trial_url) process_start_time = datetime.now() # trigger reset of timer for last processed trial processTrial(trial["session"], trial["id"], trial_type=trial_type, isDocker=isDocker) + + # End process duration timer and post duration to database process_end_time = datetime.now() postProcessedDuration(trial_url, process_end_time - process_start_time) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 87223113..dce06d31 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,7 +1,7 @@ version: "3.9" services: mobilecap: - image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/opencap-dev + image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/opencap volumes: - data:/data env_file: @@ -16,7 +16,7 @@ services: count: 1 capabilities: [gpu] openpose: - image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/openpose-dev + image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/openpose volumes: - data:/openpose/data deploy: @@ -27,7 +27,7 @@ services: count: 1 capabilities: [gpu] mmpose: - image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/mmpose-dev + image: 660440363484.dkr.ecr.us-west-2.amazonaws.com/opencap/mmpose volumes: - data:/mmpose/data deploy: diff --git a/utils.py b/utils.py index 5e071482..8750a351 100644 --- a/utils.py +++ b/utils.py @@ -1617,12 +1617,20 @@ def get_entry_with_largest_number(trialList): # Get local client info and update def getCommitHash(): + """Get the git commit hash stored in the environment variable + GIT_COMMIT_HASH. This is assumed to be set in the Docker build + step. If not set, returns Null (default value for os.getenv()) + """ return os.getenv('GIT_COMMIT_HASH') def getHostname(): + """Get the hostname. For a docker container, this is the container ID.""" return socket.gethostname() def postLocalClientInfo(trial_url): + """Given a trial_url, updates the Trial fields for + 'git_commit' and 'hostname'. + """ data = { "git_commit": getCommitHash(), "hostname": getHostname() @@ -1633,6 +1641,9 @@ def postLocalClientInfo(trial_url): return r def postProcessedDuration(trial_url, duration): + """Given a trial_url and duration (formed from difference in datetime + objects), updates the Trial field for 'processed_duration'. + """ data = { "processed_duration": duration }