TES Support: Implementing adapter-service for TES

cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java

@@ -16,6 +16,7 @@
 
 package com.epam.pipeline.client.pipeline;
 
+import com.epam.pipeline.entity.cluster.AllowedInstanceAndPriceTypes;
 import com.epam.pipeline.entity.cluster.NodeInstance;
 import com.epam.pipeline.entity.configuration.RunConfiguration;
 import com.epam.pipeline.entity.datastorage.AbstractDataStorage;
@@ -30,19 +31,24 @@
 import com.epam.pipeline.entity.pipeline.Folder;
 import com.epam.pipeline.entity.pipeline.Pipeline;
 import com.epam.pipeline.entity.pipeline.PipelineRun;
+import com.epam.pipeline.entity.pipeline.PipelineTask;
 import com.epam.pipeline.entity.pipeline.Revision;
 import com.epam.pipeline.entity.pipeline.RunInstance;
 import com.epam.pipeline.entity.pipeline.RunLog;
 import com.epam.pipeline.entity.pipeline.Tool;
 import com.epam.pipeline.entity.pipeline.ToolGroup;
+import com.epam.pipeline.entity.pipeline.run.PipelineStart;
 import com.epam.pipeline.entity.region.AbstractCloudRegion;
 import com.epam.pipeline.entity.region.AwsRegion;
 import com.epam.pipeline.entity.security.acl.AclClass;
 import com.epam.pipeline.entity.user.PipelineUser;
+import com.epam.pipeline.rest.PagedResult;
 import com.epam.pipeline.rest.Result;
 import com.epam.pipeline.vo.EntityPermissionVO;
 import com.epam.pipeline.vo.EntityVO;
 import com.epam.pipeline.vo.FilterNodesVO;
+import com.epam.pipeline.vo.PagingRunFilterExpressionVO;
+import com.epam.pipeline.vo.PagingRunFilterVO;
 import com.epam.pipeline.vo.RunStatusVO;
 import com.epam.pipeline.vo.notification.NotificationMessageVO;
 import okhttp3.MultipartBody;
@@ -71,6 +77,21 @@ public interface CloudPipelineAPI {
     String TOOL_IDENTIFIER = "image";
     String VERSION = "version";
     String PATH = "path";
+    String TOOL_ID = "toolId";
+    String REGION_ID = "regionId";
+    String SPOT = "spot";
+    String LOAD_LINKS = "loadLinks";
+
+
+    @POST("run")
+    Call<Result<PipelineRun>> runPipeline(@Body PipelineStart runVo);
+
+    @POST("run/search")
+    Call<Result<PagedResult<List<PipelineRun>>>> searchRuns(@Body PagingRunFilterExpressionVO filterVO);
+
+    @POST("run/filter")
+    Call<Result<PagedResult<List<PipelineRun>>>> filterRuns(@Body PagingRunFilterVO filterVO,
+                                                            @Query(LOAD_LINKS) Boolean loadStorageLinks);
 
     @POST("run/{runId}/status")
     Call<Result<PipelineRun>> updateRunStatus(@Path(RUN_ID) Long runId,
@@ -189,7 +210,15 @@ Call<Result<List<GitRepositoryEntry>>> loadRepositoryContent(@Path(ID) Long id,
     @POST("cluster/node/filter")
     Call<Result<List<NodeInstance>>> findNodes(@Body FilterNodesVO filterNodesVO);
 
+    @GET("cluster/instance/allowed")
+    Call<Result<AllowedInstanceAndPriceTypes>> loadAllowedInstanceAndPriceTypes(@Query(TOOL_ID) Long toolId,
+                                                                                @Query(REGION_ID) Long regionId,
+                                                                                @Query(SPOT) Boolean spot);
+
     //Notification methods
     @POST("notification/message")
     Call<Result<NotificationMessage>> createNotification(@Body NotificationMessageVO notification);
+
+    @GET("run/{runId}/tasks")
+    Call<Result<List<PipelineTask>>> loadPipelineTasks(@Path(RUN_ID) Long runId);
 }

cloud-pipeline-common/model/src/main/java/com/epam/pipeline/entity/cluster/AllowedInstanceAndPriceTypes.java

@@ -23,12 +23,12 @@
 @Value
 public class AllowedInstanceAndPriceTypes {
 
-    @JsonProperty("allowed.instance.types")
+    @JsonProperty("cluster.allowed.instance.types")
     private final List<InstanceType> allowedInstanceTypes;
 
-    @JsonProperty("allowed.instance.docker.types")
+    @JsonProperty("cluster.allowed.instance.docker.types")
     private final List<InstanceType> allowedInstanceDockerTypes;
 
-    @JsonProperty("allowed.price.types")
+    @JsonProperty("cluster.allowed.price.types")
     private final List<String> allowedPriceTypes;
 }

cloud-pipeline-common/model/src/main/java/com/epam/pipeline/entity/cluster/InstanceType.java

@@ -18,6 +18,7 @@
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
+import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
@@ -27,6 +28,7 @@
 @Builder
 @AllArgsConstructor
 @NoArgsConstructor
+@EqualsAndHashCode
 public class InstanceType {
     private String sku;
     private String name;

cloud-pipeline-common/model/src/main/java/com/epam/pipeline/rest/PagedResult.java

@@ -16,6 +16,11 @@
 
 package com.epam.pipeline.rest;
 
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Setter
+@NoArgsConstructor
 public class PagedResult<T> { // TODO: refactor to extend Result class
     private T elements; // TODO; refactor to contain a list of T
     private int totalCount;

cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.epam.pipeline.vo;
+
+import com.epam.pipeline.entity.filter.AclSecuredFilter;
+import com.epam.pipeline.vo.filter.FilterExpressionVO;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.util.List;
+
+@Getter
+@Setter
+@NoArgsConstructor
+public class PagingRunFilterExpressionVO implements AclSecuredFilter {
+
+    private int page;
+    private int pageSize;
+    private int timezoneOffsetInMinutes;
+    private FilterExpressionVO filterExpression;
+
+    //these filter is used for ACL filtering
+    @JsonIgnore
+    private List<Long> allowedPipelines;
+    @JsonIgnore
+    private String ownershipFilter;
+}

cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionTypeVO.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.epam.pipeline.vo.filter;
+
+public enum FilterExpressionTypeVO {
+    LOGICAL(0),
+    AND(1),
+    OR(2);
+
+    private long id;
+
+    FilterExpressionTypeVO(long id) {
+        this.id = id;
+    }
+
+    public Long getId() {
+        return this.id;
+    }
+}

cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionVO.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.epam.pipeline.vo.filter;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.util.List;
+
+@Getter
+@Setter
+@NoArgsConstructor
+public class FilterExpressionVO {
+
+    private String field;
+    private String value;
+    private String operand;
+
+    @JsonIgnore
+    private FilterOperandTypeVO operandType;
+
+    private List<FilterExpressionVO> expressions;
+    private FilterExpressionTypeVO filterExpressionType;
+}

cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandTypeVO.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.epam.pipeline.vo.filter;
+
+public enum FilterOperandTypeVO {
+    LESS("<"),
+    LESS_OR_EQUALS("<="),
+    EQUALS("="),
+    NOT_EQUALS("!="),
+    MORE_OR_EQUALS(">="),
+    MORE(">");
+
+    private String operand;
+
+    FilterOperandTypeVO(String operand) {
+        this.operand = operand;
+    }
+
+    public String getOperand() {
+        return this.operand;
+    }
+}

deploy/README.md

@@ -125,6 +125,9 @@ bash build.sh -aws eu-central-1,us-east-1 \                         # List of re
                 -env CP_KUBE_MASTER_ETCD_HOST_PATH= \               # Allows to override a location of the folder where etcd stores wal/data dirs. This is useful when etcd runs into I/O latency issues and shall be pointed to another device mounted to a more custom location, which leads to the kube control plane failures. If not defined - /var/lib/etcd path will be used
                 -env CP_KUBE_MIN_DNS_REPLICAS= \                    # Allows to configure a minimal number of DNS replicas for the cluster (default: 1). DNS will be autoscaled based on the size of a cluster (1 new replica for each 128 cores or 5 nodes)
                 -env CP_KUBE_SERVICES_TYPE= \                       # Allows to select a preferred services mode type: "node-port" or "external-ip" (default: "node-port")
+
+                # TES Adapter
+                -env CP_TES_WHITE_IPV4_CIDR                         # Whitelist ip addresses for access to Cloud Pipeline through TES adapter, if client doesn't have HttpAuthorization in headers or cookies. It supports as IPv4 as Ipv6  
 ```
 
 # Examples

deploy/contents/install/app/install.sh

@@ -243,6 +243,11 @@ CP_SHARE_SRV_KUBE_NODE_NAME=${CP_SHARE_SRV_KUBE_NODE_NAME:-$KUBE_MASTER_NODE_NAM
 print_info "-> Assigning cloud-pipeline/cp-share-srv to $CP_SHARE_SRV_KUBE_NODE_NAME"
 kubectl label nodes "$CP_SHARE_SRV_KUBE_NODE_NAME" cloud-pipeline/cp-share-srv="true" --overwrite
 
+# Allow to schedule TES service to the master
+CP_TES_SRV_KUBE_NODE_NAME=${CP_TES_SRV_KUBE_NODE_NAME:-$KUBE_MASTER_NODE_NAME}
+print_info "-> Assigning cloud-pipeline/cp-tes-srv to $CP_TES_SRV_KUBE_NODE_NAME"
+kubectl label nodes "$CP_TES_SRV_KUBE_NODE_NAME" cloud-pipeline/cp-tes-srv="true" --overwrite
+
 echo
 
 ##########
@@ -502,7 +507,7 @@ if is_service_requested cp-api-srv; then
         print_info "-> Registering custom users in IdP and API services"
         api_register_custom_users "$CP_CUSTOM_USERS_SPEC"
 
-        # Here we wait for the price list sync, this is required by the downstream services to manage the instance types configurations            
+        # Here we wait for the price list sync, this is required by the downstream services to manage the instance types configurations
         if [ -z "$CP_CLOUD_REGION_INTERNAL_ID" ]; then
             print_warn "CP_CLOUD_REGION_INTERNAL_ID is not defined, assuming that a cloud region is not registered correctly previously. WILL NOT wait for price lists synchonization"
         else
@@ -607,7 +612,7 @@ if is_service_requested cp-edge; then
     delete_deployment_and_service   "cp-edge" \
                                     "/opt/edge"
 
-    if is_install_requested; then   
+    if is_install_requested; then
         print_info "-> Creating self-signed SSL certificate for EDGE (${CP_EDGE_EXTERNAL_HOST}, ${CP_EDGE_INTERNAL_HOST})"
         generate_self_signed_key_pair   $CP_EDGE_CERT_DIR/ssl-private-key.pem \
                                         $CP_EDGE_CERT_DIR/ssl-public-cert.pem \
@@ -628,7 +633,7 @@ if is_service_requested cp-edge; then
         update_config_value "$CP_INSTALL_CONFIG_FILE" \
                                 "EDGE_EXTERNAL" \
                                 "$EDGE_EXTERNAL"
-        
+
         init_kube_config_map
 
         print_ok "-> EDGE addresses parameters set:"
@@ -917,7 +922,7 @@ if is_service_requested cp-search; then
 
     print_info "-> Deleting existing instance of Search ELK service"
     delete_deployment_and_service   "cp-search-elk" \
-                                    "/opt/search-elk"    
+                                    "/opt/search-elk"
 
     if is_install_requested; then
         print_info "-> Deploying Search ELK service"
@@ -1050,6 +1055,25 @@ if is_service_requested cp-share-srv; then
     echo
 fi
 
+# TES Service
+if is_service_requested cp-tes-srv; then
+print_ok "[Starting TES Service deployment]"
+
+    print_info "-> Deleting existing instance of TES Service"
+    delete_deployment_and_service   "cp-tes-srv" \
+                                    "/opt/tes-srv"
+    if is_install_requested; then
+        print_info "-> Deploying TES service"
+        create_kube_resource $K8S_SPECS_HOME/cp-tes-srv/cp-tes-srv-dpl.yaml
+        create_kube_resource $K8S_SPECS_HOME/cp-tes-srv/cp-tes-srv-svc.yaml --svc
+
+        print_info "-> Waiting for TES service to initialize"
+        wait_for_deployment "cp-tes-srv"
+
+        CP_INSTALL_SUMMARY="$CP_INSTALL_SUMMARY\ncp-tes-srv: deployed"
+    fi
+    echo
+fi
 
 print_ok "Installation done"
 echo -e $CP_INSTALL_SUMMARY

deploy/contents/install/install-config

@@ -154,5 +154,8 @@ CP_DAV_SERVE_DIR=/dav-serve
 CP_DAV_MOUNT_POINT=/dav-mount
 CP_DAV_INSTRUCTIONS_URL=
 
+# cp-tes-srv
+CP_TES_WHITE_IPV4_CIDR=127.0.0.1/24
+
 # Other params, specified on command line
 #

deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml

@@ -0,0 +1,28 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: cp-tes-srv
+  namespace: default
+spec:
+  replicas: 1
+  template:
+    metadata:
+      name: cp-tes-srv
+      namespace: default
+      labels:
+        cloud-pipeline/cp-tes-srv: "true"
+    spec:
+      nodeSelector:
+        cloud-pipeline/cp-tes-srv: "true"
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      containers:
+        - name: cp-tes-srv
+          image: lifescience/cloud-pipeline:tes-srv-$CP_VERSION
+          imagePullPolicy: "IfNotPresent"
+          ports:
+            - containerPort: 8080
+          envFrom:
+            - configMapRef:
+                name: cp-config-global