-
Notifications
You must be signed in to change notification settings - Fork 122
/
Makefile
196 lines (176 loc) · 6.21 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
GIT_HEAD_REF := $(shell git rev-parse HEAD)
BASE_IMAGE := pytorch/pytorch:1.9.0-cuda11.1-cudnn8-devel
DEV_IMAGE_NAME := text-to-sql-dev
TRAIN_IMAGE_NAME := text-to-sql-train
EVAL_IMAGE_NAME := text-to-sql-eval
BUILDKIT_IMAGE := tscholak/text-to-sql-buildkit:buildx-stable-1
BUILDKIT_BUILDER ?= buildx-local
BASE_DIR := $(shell pwd)
.PHONY: init-buildkit
init-buildkit:
docker buildx create \
--name buildx-local \
--driver docker-container \
--driver-opt image=$(BUILDKIT_IMAGE),network=host \
--use
.PHONY: del-buildkit
del-buildkit:
docker buildx rm buildx-local
.PHONY: build-thrift-code
build-thrift-code:
thrift1 --gen mstch_cpp2 picard.thrift
thrift1 --gen mstch_py3 picard.thrift
cd gen-py3 && python setup.py build_ext --inplace
.PHONY: build-picard-deps
build-picard-deps:
cabal update
thrift-compiler --hs --use-hash-map --use-hash-set --gen-prefix gen-hs -o . picard.thrift
patch -p 1 -N -d third_party/hsthrift < ./fb-util-cabal.patch || true
cd third_party/hsthrift \
&& make THRIFT_COMPILE=thrift-compiler thrift-cpp thrift-hs
cabal build --only-dependencies lib:picard
.PHONY: build-picard
build-picard:
cabal install --overwrite-policy=always --install-method=copy exe:picard
.PHONY: build-dev-image
build-dev-image:
ssh-add
docker buildx build \
--builder $(BUILDKIT_BUILDER) \
--ssh default=$(SSH_AUTH_SOCK) \
-f Dockerfile \
--tag tscholak/$(DEV_IMAGE_NAME):$(GIT_HEAD_REF) \
--tag tscholak/$(DEV_IMAGE_NAME):cache \
--tag tscholak/$(DEV_IMAGE_NAME):devcontainer \
--build-arg BASE_IMAGE=$(BASE_IMAGE) \
--target dev \
--cache-from type=registry,ref=tscholak/$(DEV_IMAGE_NAME):cache \
--cache-to type=inline \
--push \
[email protected]:ElementAI/picard#$(GIT_HEAD_REF)
.PHONY: pull-dev-image
pull-dev-image:
docker pull tscholak/$(DEV_IMAGE_NAME):$(GIT_HEAD_REF)
.PHONY: build-train-image
build-train-image:
ssh-add
docker buildx build \
--builder $(BUILDKIT_BUILDER) \
--ssh default=$(SSH_AUTH_SOCK) \
-f Dockerfile \
--tag tscholak/$(TRAIN_IMAGE_NAME):$(GIT_HEAD_REF) \
--tag tscholak/$(TRAIN_IMAGE_NAME):cache \
--build-arg BASE_IMAGE=$(BASE_IMAGE) \
--target train \
--cache-from type=registry,ref=tscholak/$(TRAIN_IMAGE_NAME):cache \
--cache-to type=inline \
--push \
[email protected]:ElementAI/picard#$(GIT_HEAD_REF)
.PHONY: pull-train-image
pull-train-image:
docker pull tscholak/$(TRAIN_IMAGE_NAME):$(GIT_HEAD_REF)
.PHONY: build-eval-image
build-eval-image:
ssh-add
docker buildx build \
--builder $(BUILDKIT_BUILDER) \
--ssh default=$(SSH_AUTH_SOCK) \
-f Dockerfile \
--tag tscholak/$(EVAL_IMAGE_NAME):$(GIT_HEAD_REF) \
--tag tscholak/$(EVAL_IMAGE_NAME):cache \
--build-arg BASE_IMAGE=$(BASE_IMAGE) \
--target eval \
--cache-from type=registry,ref=tscholak/$(EVAL_IMAGE_NAME):cache \
--cache-to type=inline \
--push \
[email protected]:ElementAI/picard#$(GIT_HEAD_REF)
.PHONY: pull-eval-image
pull-eval-image:
docker pull tscholak/$(EVAL_IMAGE_NAME):$(GIT_HEAD_REF)
.PHONY: train
train: pull-train-image
mkdir -p -m 777 train
mkdir -p -m 777 transformers_cache
mkdir -p -m 777 wandb
docker run \
-it \
--rm \
--user 13011:13011 \
--mount type=bind,source=$(BASE_DIR)/train,target=/train \
--mount type=bind,source=$(BASE_DIR)/transformers_cache,target=/transformers_cache \
--mount type=bind,source=$(BASE_DIR)/configs,target=/app/configs \
--mount type=bind,source=$(BASE_DIR)/wandb,target=/app/wandb \
tscholak/$(TRAIN_IMAGE_NAME):$(GIT_HEAD_REF) \
/bin/bash -c "python seq2seq/run_seq2seq.py configs/train.json"
.PHONY: train_cosql
train_cosql: pull-train-image
mkdir -p -m 777 train
mkdir -p -m 777 transformers_cache
mkdir -p -m 777 wandb
docker run \
-it \
--rm \
--user 13011:13011 \
--mount type=bind,source=$(BASE_DIR)/train,target=/train \
--mount type=bind,source=$(BASE_DIR)/transformers_cache,target=/transformers_cache \
--mount type=bind,source=$(BASE_DIR)/configs,target=/app/configs \
--mount type=bind,source=$(BASE_DIR)/wandb,target=/app/wandb \
tscholak/$(TRAIN_IMAGE_NAME):$(GIT_HEAD_REF) \
/bin/bash -c "python seq2seq/run_seq2seq.py configs/train_cosql.json"
.PHONY: eval
eval: pull-eval-image
mkdir -p -m 777 eval
mkdir -p -m 777 transformers_cache
mkdir -p -m 777 wandb
docker run \
-it \
--rm \
--user 13011:13011 \
--mount type=bind,source=$(BASE_DIR)/eval,target=/eval \
--mount type=bind,source=$(BASE_DIR)/transformers_cache,target=/transformers_cache \
--mount type=bind,source=$(BASE_DIR)/configs,target=/app/configs \
--mount type=bind,source=$(BASE_DIR)/wandb,target=/app/wandb \
tscholak/$(EVAL_IMAGE_NAME):$(GIT_HEAD_REF) \
/bin/bash -c "python seq2seq/run_seq2seq.py configs/eval.json"
.PHONY: eval_cosql
eval_cosql: pull-eval-image
mkdir -p -m 777 eval
mkdir -p -m 777 transformers_cache
mkdir -p -m 777 wandb
docker run \
-it \
--rm \
--user 13011:13011 \
--mount type=bind,source=$(BASE_DIR)/eval,target=/eval \
--mount type=bind,source=$(BASE_DIR)/transformers_cache,target=/transformers_cache \
--mount type=bind,source=$(BASE_DIR)/configs,target=/app/configs \
--mount type=bind,source=$(BASE_DIR)/wandb,target=/app/wandb \
tscholak/$(EVAL_IMAGE_NAME):$(GIT_HEAD_REF) \
/bin/bash -c "python seq2seq/run_seq2seq.py configs/eval_cosql.json"
.PHONY: serve
serve: pull-eval-image
mkdir -p -m 777 database
mkdir -p -m 777 transformers_cache
docker run \
-it \
--rm \
--user 13011:13011 \
-p 8000:8000 \
--mount type=bind,source=$(BASE_DIR)/database,target=/database \
--mount type=bind,source=$(BASE_DIR)/transformers_cache,target=/transformers_cache \
--mount type=bind,source=$(BASE_DIR)/configs,target=/app/configs \
tscholak/$(EVAL_IMAGE_NAME):$(GIT_HEAD_REF) \
/bin/bash -c "python seq2seq/serve_seq2seq.py configs/serve.json"
.PHONY: prediction_output
prediction_output: pull-eval-image
mkdir -p -m 777 prediction_output
docker run \
-it \
--rm \
--user 13011:13011 \
-p 8000:8000 \
--mount type=bind,source=$(BASE_DIR)/prediction_output,target=/prediction_output \
--mount type=bind,source=$(BASE_DIR)/transformers_cache,target=/transformers_cache \
--mount type=bind,source=$(BASE_DIR)/configs,target=/app/configs \
tscholak/$(EVAL_IMAGE_NAME):$(GIT_HEAD_REF) \
/bin/bash -c "python seq2seq/prediction_output.py configs/prediction_output.json"