From 858a19efd64cb5f1ed97522cb2d527ef2a5f4425 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Sun, 11 Aug 2019 22:49:12 +0300 Subject: [PATCH 001/194] add tes-adapter --- settings.gradle | 4 +++- tes-adapter/build.gradle | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tes-adapter/build.gradle diff --git a/settings.gradle b/settings.gradle index 8fbf03f3bc..5734c1cd7a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -13,4 +13,6 @@ include 'data-transfer-service' include 'vm-monitor' include 'data-sharing-service' include 'data-sharing-service:api' -include 'data-sharing-service:client' \ No newline at end of file +include 'data-sharing-service:client' +include 'tes-adapter' + diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle new file mode 100644 index 0000000000..cedb57e37c --- /dev/null +++ b/tes-adapter/build.gradle @@ -0,0 +1,15 @@ +plugins { + id 'java' +} + +group 'com.epam.pipeline' + +sourceCompatibility = 1.8 + +repositories { + mavenCentral() +} + +dependencies { + testCompile group: 'junit', name: 'junit', version: '4.12' +} From e554bcafd312b1384fc5e67b386d053fa332e9ea Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Mon, 12 Aug 2019 20:10:40 +0300 Subject: [PATCH 002/194] create feature branche and commit it to there --- tes-adapter/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index cedb57e37c..649f0520af 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -12,4 +12,5 @@ repositories { dependencies { testCompile group: 'junit', name: 'junit', version: '4.12' + } From 056ec689f3c24c7c76000bc3af24312dc86aa839 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 13 Aug 2019 16:04:08 +0300 Subject: [PATCH 003/194] simple-web-application implemented. sub-task 2759 --- tes-adapter/build.gradle | 35 +++++++++++++++---- tes-adapter/src/main/application.properties | 0 .../adaptor/TesAdapterApplication.java | 4 +++ .../controller/TesAdapterController.java | 4 +++ 4 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 tes-adapter/src/main/application.properties create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/adaptor/TesAdapterApplication.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/adaptor/controller/TesAdapterController.java diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index 649f0520af..1cc109fc80 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -1,16 +1,37 @@ -plugins { - id 'java' -} +group 'com.epam.pipeline.adapter' +version '1.0-SNAPSHOT' -group 'com.epam.pipeline' +buildscript { + ext { + springBootVersion = '1.5.2.RELEASE' + } + repositories { + mavenCentral() + } + dependencies { + classpath("org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}") + } +} -sourceCompatibility = 1.8 +apply plugin: 'java' +apply plugin: 'org.springframework.boot' repositories { mavenCentral() } +jar { + manifest { + attributes 'Main-Class': 'com.epam.pipeline.adapter.TesAdapterApplication' + } + from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } } +} + +sourceCompatibility = 1.8 + dependencies { + compile('org.springframework.boot:spring-boot-starter-web') + compileOnly 'org.projectlombok:lombok:1.18.8' + annotationProcessor 'org.projectlombok:lombok:1.18.8' testCompile group: 'junit', name: 'junit', version: '4.12' - -} +} \ No newline at end of file diff --git a/tes-adapter/src/main/application.properties b/tes-adapter/src/main/application.properties new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tes-adapter/src/main/java/com/epam/pipeline/adaptor/TesAdapterApplication.java b/tes-adapter/src/main/java/com/epam/pipeline/adaptor/TesAdapterApplication.java new file mode 100644 index 0000000000..6cfe10aaf1 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/adaptor/TesAdapterApplication.java @@ -0,0 +1,4 @@ +package com.epam.pipeline.adaptor; + +public class TesAdapterApplication { +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/adaptor/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/adaptor/controller/TesAdapterController.java new file mode 100644 index 0000000000..5fad282b5f --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/adaptor/controller/TesAdapterController.java @@ -0,0 +1,4 @@ +package com.epam.pipeline.adaptor.controller; + +public class TesAdapterController { +} From 771c9d79d932416eace76b1a9b723da85473778a Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 13 Aug 2019 16:16:35 +0300 Subject: [PATCH 004/194] simple-web-application upgraded. sub-task 2757-2759 completed --- .../com/epam/pipeline/adaptor/TesAdapterApplication.java | 7 +++++++ .../adaptor/controller/TesAdapterController.java | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/adaptor/TesAdapterApplication.java b/tes-adapter/src/main/java/com/epam/pipeline/adaptor/TesAdapterApplication.java index 6cfe10aaf1..b335f4cda7 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/adaptor/TesAdapterApplication.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/adaptor/TesAdapterApplication.java @@ -1,4 +1,11 @@ package com.epam.pipeline.adaptor; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication public class TesAdapterApplication { + public static void main(String[] args) { + SpringApplication.run(TesAdapterApplication.class, args); + } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/adaptor/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/adaptor/controller/TesAdapterController.java index 5fad282b5f..fc16eaa4fe 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/adaptor/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/adaptor/controller/TesAdapterController.java @@ -1,4 +1,13 @@ package com.epam.pipeline.adaptor.controller; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController public class TesAdapterController { + + @GetMapping("/v1/tasks/service-info") + public String getSimpleResponse(){ + return "OK"; + } } From 98497a8f92f6bf017aa99da303093562519340f4 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 15 Aug 2019 14:03:30 +0300 Subject: [PATCH 005/194] simple-web-application upgraded. fixed last pull-request mistakes, added WebConfiguration class and simple ResponseEntity<> implementation --- tes-adapter/build.gradle | 4 ++-- .../controller/TesAdapterController.java | 13 ------------ .../TesAdapterApplication.java | 5 ++++- .../TesAdapterWebConfiguration.java | 10 ++++++++++ .../controller/TesAdapterController.java | 20 +++++++++++++++++++ 5 files changed, 36 insertions(+), 16 deletions(-) delete mode 100644 tes-adapter/src/main/java/com/epam/pipeline/adaptor/controller/TesAdapterController.java rename tes-adapter/src/main/java/com/epam/pipeline/{adaptor => tesadapter}/TesAdapterApplication.java (58%) create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesAdapterWebConfiguration.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index 1cc109fc80..9be53f5805 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -1,4 +1,4 @@ -group 'com.epam.pipeline.adapter' +group 'com.epam.pipeline' version '1.0-SNAPSHOT' buildscript { @@ -22,7 +22,7 @@ repositories { jar { manifest { - attributes 'Main-Class': 'com.epam.pipeline.adapter.TesAdapterApplication' + attributes 'Main-Class': 'com.epam.pipeline.tesadapter.TesAdapterApplication' } from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/adaptor/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/adaptor/controller/TesAdapterController.java deleted file mode 100644 index fc16eaa4fe..0000000000 --- a/tes-adapter/src/main/java/com/epam/pipeline/adaptor/controller/TesAdapterController.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.epam.pipeline.adaptor.controller; - -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RestController; - -@RestController -public class TesAdapterController { - - @GetMapping("/v1/tasks/service-info") - public String getSimpleResponse(){ - return "OK"; - } -} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/adaptor/TesAdapterApplication.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java similarity index 58% rename from tes-adapter/src/main/java/com/epam/pipeline/adaptor/TesAdapterApplication.java rename to tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java index b335f4cda7..3e2ed60572 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/adaptor/TesAdapterApplication.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java @@ -1,9 +1,12 @@ -package com.epam.pipeline.adaptor; +package com.epam.pipeline.tesadapter; +import com.epam.pipeline.tesadapter.configuration.TesAdapterWebConfiguration; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Import; @SpringBootApplication +@Import({TesAdapterWebConfiguration.class}) public class TesAdapterApplication { public static void main(String[] args) { SpringApplication.run(TesAdapterApplication.class, args); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesAdapterWebConfiguration.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesAdapterWebConfiguration.java new file mode 100644 index 0000000000..88754ca14c --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesAdapterWebConfiguration.java @@ -0,0 +1,10 @@ +package com.epam.pipeline.tesadapter.configuration; + +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; + +@EnableWebMvc +@Configuration +public class TesAdapterWebConfiguration extends WebMvcConfigurerAdapter { +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java new file mode 100644 index 0000000000..42e1f6b494 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -0,0 +1,20 @@ +package com.epam.pipeline.tesadapter.controller; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class TesAdapterController { + + @GetMapping("/v1/tasks/service-info") + public ResponseEntity serviceInfo( + @RequestParam(name = "status", required = false, defaultValue = "OK") String status){ + if (status.equalsIgnoreCase("ok")){ + return new ResponseEntity<>("The system is fully operational!", HttpStatus.OK); + } + return new ResponseEntity<>("The system is not operable!", HttpStatus.NOT_FOUND); + } +} From 9928553251faeac367f1a8e066e4453a3597c83d Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 15 Aug 2019 15:05:07 +0300 Subject: [PATCH 006/194] simple-web-application upgraded. fixed TesAdapterController, added 'OK' response to "/v1/tasks/service-info" endpoint --- .../tesadapter/controller/TesAdapterController.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index 42e1f6b494..2effbf7cc6 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -10,11 +10,7 @@ public class TesAdapterController { @GetMapping("/v1/tasks/service-info") - public ResponseEntity serviceInfo( - @RequestParam(name = "status", required = false, defaultValue = "OK") String status){ - if (status.equalsIgnoreCase("ok")){ - return new ResponseEntity<>("The system is fully operational!", HttpStatus.OK); - } - return new ResponseEntity<>("The system is not operable!", HttpStatus.NOT_FOUND); + public ResponseEntity serviceInfo() { + return new ResponseEntity<>("OK", HttpStatus.OK); } } From 44187e6aaac455bc8750c63b358335e141b4430e Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 15 Aug 2019 17:35:58 +0300 Subject: [PATCH 007/194] simple swagger implementation completed. added swagger configuration and couple changes in controller --- tes-adapter/build.gradle | 2 + .../tesadapter/TesAdapterApplication.java | 4 +- .../TesAdapterWebConfiguration.java | 10 ----- .../configuration/TesSwaggerConfig.java | 37 +++++++++++++++++++ .../controller/TesAdapterController.java | 7 ++-- 5 files changed, 44 insertions(+), 16 deletions(-) delete mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesAdapterWebConfiguration.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index 9be53f5805..d1e6fc80b6 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -31,6 +31,8 @@ sourceCompatibility = 1.8 dependencies { compile('org.springframework.boot:spring-boot-starter-web') + compile('io.springfox:springfox-swagger2:2.9.2') + compile('io.springfox:springfox-swagger-ui:2.9.2') compileOnly 'org.projectlombok:lombok:1.18.8' annotationProcessor 'org.projectlombok:lombok:1.18.8' testCompile group: 'junit', name: 'junit', version: '4.12' diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java index 3e2ed60572..43925d2d73 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java @@ -1,12 +1,12 @@ package com.epam.pipeline.tesadapter; -import com.epam.pipeline.tesadapter.configuration.TesAdapterWebConfiguration; +import com.epam.pipeline.tesadapter.configuration.TesSwaggerConfig; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.Import; @SpringBootApplication -@Import({TesAdapterWebConfiguration.class}) +@Import({TesSwaggerConfig.class}) public class TesAdapterApplication { public static void main(String[] args) { SpringApplication.run(TesAdapterApplication.class, args); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesAdapterWebConfiguration.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesAdapterWebConfiguration.java deleted file mode 100644 index 88754ca14c..0000000000 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesAdapterWebConfiguration.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.epam.pipeline.tesadapter.configuration; - -import org.springframework.context.annotation.Configuration; -import org.springframework.web.servlet.config.annotation.EnableWebMvc; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; - -@EnableWebMvc -@Configuration -public class TesAdapterWebConfiguration extends WebMvcConfigurerAdapter { -} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java new file mode 100644 index 0000000000..40e6202966 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java @@ -0,0 +1,37 @@ +package com.epam.pipeline.tesadapter.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import springfox.documentation.builders.ApiInfoBuilder; +import springfox.documentation.builders.PathSelectors; +import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.service.ApiInfo; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spring.web.plugins.Docket; +import springfox.documentation.swagger2.annotations.EnableSwagger2; + + +@Configuration +@EnableSwagger2 +public class TesSwaggerConfig { + + @Bean + public Docket api() { + return new Docket(DocumentationType.SWAGGER_2) + .apiInfo(getApiInfo()) + .select() + .apis(RequestHandlerSelectors.any()) + .paths(PathSelectors.any()) + .build(); + } + + + private ApiInfo getApiInfo() { + return new ApiInfoBuilder() + .title("TES adapter API Documentation") + .description("Simple TES adapter application. " + + "Without Security Configuration and Context") + .version("1.0.0") + .build(); + } +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index 2effbf7cc6..a26796da5a 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -2,14 +2,13 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; @RestController public class TesAdapterController { - - @GetMapping("/v1/tasks/service-info") + @RequestMapping(method = RequestMethod.GET, value = "/v1/tasks/service-info") public ResponseEntity serviceInfo() { return new ResponseEntity<>("OK", HttpStatus.OK); } From d8be81093cd54ea9e41b99ef07016f57585fc32e Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 15 Aug 2019 19:15:15 +0300 Subject: [PATCH 008/194] simple swagger implementation fixed. returned @GetMapping annotation. Configuring swagger paths to controller package and also tasks --- tes-adapter/build.gradle | 10 +++++----- .../tesadapter/configuration/TesSwaggerConfig.java | 4 ++-- .../tesadapter/controller/TesAdapterController.java | 5 ++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index d1e6fc80b6..b89368eb5d 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -2,19 +2,18 @@ group 'com.epam.pipeline' version '1.0-SNAPSHOT' buildscript { - ext { - springBootVersion = '1.5.2.RELEASE' - } repositories { mavenCentral() } dependencies { - classpath("org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}") + classpath("org.springframework.boot:spring-boot-gradle-plugin:2.1.6.RELEASE") } } apply plugin: 'java' +apply plugin: 'idea' apply plugin: 'org.springframework.boot' +apply plugin: 'io.spring.dependency-management' repositories { mavenCentral() @@ -35,5 +34,6 @@ dependencies { compile('io.springfox:springfox-swagger-ui:2.9.2') compileOnly 'org.projectlombok:lombok:1.18.8' annotationProcessor 'org.projectlombok:lombok:1.18.8' - testCompile group: 'junit', name: 'junit', version: '4.12' + testImplementation('org.junit.jupiter:junit-jupiter-api:5.4.2') + testRuntime('org.junit.jupiter:junit-jupiter-engine:5.4.2') } \ No newline at end of file diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java index 40e6202966..6c2887e3cb 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java @@ -20,8 +20,8 @@ public Docket api() { return new Docket(DocumentationType.SWAGGER_2) .apiInfo(getApiInfo()) .select() - .apis(RequestHandlerSelectors.any()) - .paths(PathSelectors.any()) + .apis(RequestHandlerSelectors.basePackage("com.epam.pipeline.tesadapter.controller")) + .paths(PathSelectors.ant("/v1/tasks/**")) .build(); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index a26796da5a..be83a8b6ec 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -2,13 +2,12 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; @RestController public class TesAdapterController { - @RequestMapping(method = RequestMethod.GET, value = "/v1/tasks/service-info") + @GetMapping("/v1/tasks/service-info") public ResponseEntity serviceInfo() { return new ResponseEntity<>("OK", HttpStatus.OK); } From 2ac73ebb365143c4f146c8e4084ea7a051788a41 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 16 Aug 2019 19:45:05 +0300 Subject: [PATCH 009/194] simple test tasks service implemented. injected to TesAdapterController and also in ComponentScan --- .../epam/pipeline/tesadapter/TesAdapterApplication.java | 2 ++ .../tesadapter/controller/TesAdapterController.java | 5 +++++ .../epam/pipeline/tesadapter/service/TesTaskService.java | 7 +++++++ 3 files changed, 14 insertions(+) create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java index 43925d2d73..9a04a3c380 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java @@ -3,10 +3,12 @@ import com.epam.pipeline.tesadapter.configuration.TesSwaggerConfig; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Import; @SpringBootApplication @Import({TesSwaggerConfig.class}) +@ComponentScan({"com.epam.pipeline.tesadapter.controller", "com.epam.pipeline.tesadapter.service"}) public class TesAdapterApplication { public static void main(String[] args) { SpringApplication.run(TesAdapterApplication.class, args); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index be83a8b6ec..ce42a18220 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -1,5 +1,7 @@ package com.epam.pipeline.tesadapter.controller; +import com.epam.pipeline.tesadapter.service.TesTaskService; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; @@ -7,6 +9,9 @@ @RestController public class TesAdapterController { + @Autowired + private TesTaskService tesTaskService; + @GetMapping("/v1/tasks/service-info") public ResponseEntity serviceInfo() { return new ResponseEntity<>("OK", HttpStatus.OK); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java new file mode 100644 index 0000000000..2029596802 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -0,0 +1,7 @@ +package com.epam.pipeline.tesadapter.service; + +import org.springframework.stereotype.Service; + +@Service +public class TesTaskService { +} From 988a6aa0e6defe61e54438db9300a2561b81f7e6 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Sat, 17 Aug 2019 00:34:48 +0300 Subject: [PATCH 010/194] implemented task entities with swagger-codegen and task_execution.swagger.json --- .../entity/TesCancelTaskResponse.java | 54 +++ .../entity/TesCreateTaskResponse.java | 79 ++++ .../tesadapter/entity/TesExecutor.java | 252 +++++++++++ .../tesadapter/entity/TesExecutorLog.java | 179 ++++++++ .../tesadapter/entity/TesFileType.java | 38 ++ .../pipeline/tesadapter/entity/TesInput.java | 206 +++++++++ .../entity/TesListTasksResponse.java | 118 +++++ .../pipeline/tesadapter/entity/TesOutput.java | 181 ++++++++ .../tesadapter/entity/TesOutputFileLog.java | 129 ++++++ .../tesadapter/entity/TesResources.java | 191 ++++++++ .../tesadapter/entity/TesServiceInfo.java | 141 ++++++ .../pipeline/tesadapter/entity/TesState.java | 52 +++ .../pipeline/tesadapter/entity/TesTask.java | 420 ++++++++++++++++++ .../tesadapter/entity/TesTaskLog.java | 248 +++++++++++ 14 files changed, 2288 insertions(+) create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java new file mode 100644 index 0000000000..3ae223da8d --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java @@ -0,0 +1,54 @@ +package com.epam.pipeline.tesadapter.entity; + +import java.util.Objects; +import io.swagger.annotations.ApiModel; +import org.springframework.validation.annotation.Validated; +import javax.validation.Valid; +import javax.validation.constraints.*; + +/** + * CancelTaskResponse describes a response from the CancelTask endpoint. + */ +@ApiModel(description = "CancelTaskResponse describes a response from the CancelTask endpoint.") +@Validated +@javax.annotation.Generated(value = "io.swagger.codegen.languages.SpringCodegen", date = "2019-08-17T00:22:00.237+03:00") + +public class TesCancelTaskResponse { + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return Objects.hash(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class TesCancelTaskResponse {\n"); + + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java new file mode 100644 index 0000000000..28baf0a1f5 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java @@ -0,0 +1,79 @@ +package com.epam.pipeline.tesadapter.entity; + +import java.util.Objects; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import org.springframework.validation.annotation.Validated; + +/** + * CreateTaskResponse describes a response from the CreateTask endpoint. + */ +@ApiModel(description = "CreateTaskResponse describes a response from the CreateTask endpoint.") +@Validated +@javax.annotation.Generated(value = "io.swagger.codegen.languages.SpringCodegen", date = "2019-08-17T00:22:00.237+03:00") + +public class TesCreateTaskResponse { + @JsonProperty("id") + private String id = null; + + public TesCreateTaskResponse id(String id) { + this.id = id; + return this; + } + + /** + * Task identifier assigned by the server. + * @return id + **/ + @ApiModelProperty(value = "Task identifier assigned by the server.") + + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TesCreateTaskResponse tesCreateTaskResponse = (TesCreateTaskResponse) o; + return Objects.equals(this.id, tesCreateTaskResponse.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class TesCreateTaskResponse {\n"); + + sb.append(" id: ").append(toIndentedString(id)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java new file mode 100644 index 0000000000..f016f3eaed --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java @@ -0,0 +1,252 @@ +package com.epam.pipeline.tesadapter.entity; + +import java.util.Objects; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.springframework.validation.annotation.Validated; +import javax.validation.Valid; + +/** + * Executor describes a command to be executed, and its environment. + */ +@ApiModel(description = "Executor describes a command to be executed, and its environment.") +@Validated +@javax.annotation.Generated(value = "io.swagger.codegen.languages.SpringCodegen", date = "2019-08-17T00:22:00.237+03:00") + +public class TesExecutor { + @JsonProperty("image") + private String image = null; + + @JsonProperty("command") + @Valid + private List command = null; + + @JsonProperty("workdir") + private String workdir = null; + + @JsonProperty("stdin") + private String stdin = null; + + @JsonProperty("stdout") + private String stdout = null; + + @JsonProperty("stderr") + private String stderr = null; + + @JsonProperty("env") + @Valid + private Map env = null; + + public TesExecutor image(String image) { + this.image = image; + return this; + } + + /** + * Name of the container image, for example: ubuntu quay.io/aptible/ubuntu gcr.io/my-org/my-image etc... + * @return image + **/ + @ApiModelProperty(value = "Name of the container image, for example: ubuntu quay.io/aptible/ubuntu gcr.io/my-org/my-image etc...") + + + public String getImage() { + return image; + } + + public void setImage(String image) { + this.image = image; + } + + public TesExecutor command(List command) { + this.command = command; + return this; + } + + public TesExecutor addCommandItem(String commandItem) { + if (this.command == null) { + this.command = new ArrayList(); + } + this.command.add(commandItem); + return this; + } + + /** + * A sequence of program arguments to execute, where the first argument is the program to execute (i.e. argv). + * @return command + **/ + @ApiModelProperty(value = "A sequence of program arguments to execute, where the first argument is the program to execute (i.e. argv).") + + + public List getCommand() { + return command; + } + + public void setCommand(List command) { + this.command = command; + } + + public TesExecutor workdir(String workdir) { + this.workdir = workdir; + return this; + } + + /** + * The working directory that the command will be executed in. Defaults to the directory set by the container image. + * @return workdir + **/ + @ApiModelProperty(value = "The working directory that the command will be executed in. Defaults to the directory set by the container image.") + + + public String getWorkdir() { + return workdir; + } + + public void setWorkdir(String workdir) { + this.workdir = workdir; + } + + public TesExecutor stdin(String stdin) { + this.stdin = stdin; + return this; + } + + /** + * Path inside the container to a file which will be piped to the executor's stdin. Must be an absolute path. + * @return stdin + **/ + @ApiModelProperty(value = "Path inside the container to a file which will be piped to the executor's stdin. Must be an absolute path.") + + + public String getStdin() { + return stdin; + } + + public void setStdin(String stdin) { + this.stdin = stdin; + } + + public TesExecutor stdout(String stdout) { + this.stdout = stdout; + return this; + } + + /** + * Path inside the container to a file where the executor's stdout will be written to. Must be an absolute path. + * @return stdout + **/ + @ApiModelProperty(value = "Path inside the container to a file where the executor's stdout will be written to. Must be an absolute path.") + + + public String getStdout() { + return stdout; + } + + public void setStdout(String stdout) { + this.stdout = stdout; + } + + public TesExecutor stderr(String stderr) { + this.stderr = stderr; + return this; + } + + /** + * Path inside the container to a file where the executor's stderr will be written to. Must be an absolute path. + * @return stderr + **/ + @ApiModelProperty(value = "Path inside the container to a file where the executor's stderr will be written to. Must be an absolute path.") + + + public String getStderr() { + return stderr; + } + + public void setStderr(String stderr) { + this.stderr = stderr; + } + + public TesExecutor env(Map env) { + this.env = env; + return this; + } + + public TesExecutor putEnvItem(String key, String envItem) { + if (this.env == null) { + this.env = new HashMap(); + } + this.env.put(key, envItem); + return this; + } + + /** + * Enviromental variables to set within the container. + * @return env + **/ + @ApiModelProperty(value = "Enviromental variables to set within the container.") + + + public Map getEnv() { + return env; + } + + public void setEnv(Map env) { + this.env = env; + } + + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TesExecutor tesExecutor = (TesExecutor) o; + return Objects.equals(this.image, tesExecutor.image) && + Objects.equals(this.command, tesExecutor.command) && + Objects.equals(this.workdir, tesExecutor.workdir) && + Objects.equals(this.stdin, tesExecutor.stdin) && + Objects.equals(this.stdout, tesExecutor.stdout) && + Objects.equals(this.stderr, tesExecutor.stderr) && + Objects.equals(this.env, tesExecutor.env); + } + + @Override + public int hashCode() { + return Objects.hash(image, command, workdir, stdin, stdout, stderr, env); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class TesExecutor {\n"); + + sb.append(" image: ").append(toIndentedString(image)).append("\n"); + sb.append(" command: ").append(toIndentedString(command)).append("\n"); + sb.append(" workdir: ").append(toIndentedString(workdir)).append("\n"); + sb.append(" stdin: ").append(toIndentedString(stdin)).append("\n"); + sb.append(" stdout: ").append(toIndentedString(stdout)).append("\n"); + sb.append(" stderr: ").append(toIndentedString(stderr)).append("\n"); + sb.append(" env: ").append(toIndentedString(env)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java new file mode 100644 index 0000000000..4e9182805b --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java @@ -0,0 +1,179 @@ +package com.epam.pipeline.tesadapter.entity; + +import java.util.Objects; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import org.springframework.validation.annotation.Validated; + +/** + * ExecutorLog describes logging information related to an Executor. + */ +@ApiModel(description = "ExecutorLog describes logging information related to an Executor.") +@Validated +@javax.annotation.Generated(value = "io.swagger.codegen.languages.SpringCodegen", date = "2019-08-17T00:22:00.237+03:00") + +public class TesExecutorLog { + @JsonProperty("start_time") + private String startTime = null; + + @JsonProperty("end_time") + private String endTime = null; + + @JsonProperty("stdout") + private String stdout = null; + + @JsonProperty("stderr") + private String stderr = null; + + @JsonProperty("exit_code") + private Integer exitCode = null; + + public TesExecutorLog startTime(String startTime) { + this.startTime = startTime; + return this; + } + + /** + * Time the executor started, in RFC 3339 format. + * @return startTime + **/ + @ApiModelProperty(value = "Time the executor started, in RFC 3339 format.") + + + public String getStartTime() { + return startTime; + } + + public void setStartTime(String startTime) { + this.startTime = startTime; + } + + public TesExecutorLog endTime(String endTime) { + this.endTime = endTime; + return this; + } + + /** + * Time the executor ended, in RFC 3339 format. + * @return endTime + **/ + @ApiModelProperty(value = "Time the executor ended, in RFC 3339 format.") + + + public String getEndTime() { + return endTime; + } + + public void setEndTime(String endTime) { + this.endTime = endTime; + } + + public TesExecutorLog stdout(String stdout) { + this.stdout = stdout; + return this; + } + + /** + * Stdout content. This is meant for convenience. No guarantees are made about the content. Implementations may chose different approaches: only the head, only the tail, a URL reference only, etc. In order to capture the full stdout users should set Executor.stdout to a container file path, and use Task.outputs to upload that file to permanent storage. + * @return stdout + **/ + @ApiModelProperty(value = "Stdout content. This is meant for convenience. No guarantees are made about the content. Implementations may chose different approaches: only the head, only the tail, a URL reference only, etc. In order to capture the full stdout users should set Executor.stdout to a container file path, and use Task.outputs to upload that file to permanent storage.") + + + public String getStdout() { + return stdout; + } + + public void setStdout(String stdout) { + this.stdout = stdout; + } + + public TesExecutorLog stderr(String stderr) { + this.stderr = stderr; + return this; + } + + /** + * Stderr content. This is meant for convenience. No guarantees are made about the content. Implementations may chose different approaches: only the head, only the tail, a URL reference only, etc. In order to capture the full stderr users should set Executor.stderr to a container file path, and use Task.outputs to upload that file to permanent storage. + * @return stderr + **/ + @ApiModelProperty(value = "Stderr content. This is meant for convenience. No guarantees are made about the content. Implementations may chose different approaches: only the head, only the tail, a URL reference only, etc. In order to capture the full stderr users should set Executor.stderr to a container file path, and use Task.outputs to upload that file to permanent storage.") + + + public String getStderr() { + return stderr; + } + + public void setStderr(String stderr) { + this.stderr = stderr; + } + + public TesExecutorLog exitCode(Integer exitCode) { + this.exitCode = exitCode; + return this; + } + + /** + * Exit code. + * @return exitCode + **/ + @ApiModelProperty(value = "Exit code.") + + + public Integer getExitCode() { + return exitCode; + } + + public void setExitCode(Integer exitCode) { + this.exitCode = exitCode; + } + + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TesExecutorLog tesExecutorLog = (TesExecutorLog) o; + return Objects.equals(this.startTime, tesExecutorLog.startTime) && + Objects.equals(this.endTime, tesExecutorLog.endTime) && + Objects.equals(this.stdout, tesExecutorLog.stdout) && + Objects.equals(this.stderr, tesExecutorLog.stderr) && + Objects.equals(this.exitCode, tesExecutorLog.exitCode); + } + + @Override + public int hashCode() { + return Objects.hash(startTime, endTime, stdout, stderr, exitCode); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class TesExecutorLog {\n"); + + sb.append(" startTime: ").append(toIndentedString(startTime)).append("\n"); + sb.append(" endTime: ").append(toIndentedString(endTime)).append("\n"); + sb.append(" stdout: ").append(toIndentedString(stdout)).append("\n"); + sb.append(" stderr: ").append(toIndentedString(stderr)).append("\n"); + sb.append(" exitCode: ").append(toIndentedString(exitCode)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java new file mode 100644 index 0000000000..5af01a74bd --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java @@ -0,0 +1,38 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.fasterxml.jackson.annotation.JsonValue; + +import com.fasterxml.jackson.annotation.JsonCreator; + +/** + * Gets or Sets tesFileType + */ +public enum TesFileType { + + FILE("FILE"), + + DIRECTORY("DIRECTORY"); + + private String value; + + TesFileType(String value) { + this.value = value; + } + + @Override + @JsonValue + public String toString() { + return String.valueOf(value); + } + + @JsonCreator + public static TesFileType fromValue(String text) { + for (TesFileType b : TesFileType.values()) { + if (String.valueOf(b.value).equals(text)) { + return b; + } + } + return null; + } +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java new file mode 100644 index 0000000000..f683048e72 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java @@ -0,0 +1,206 @@ +package com.epam.pipeline.tesadapter.entity; + +import java.util.Objects; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import org.springframework.validation.annotation.Validated; +import javax.validation.Valid; + +/** + * Input describes Task input files. + */ +@ApiModel(description = "Input describes Task input files.") +@Validated +@javax.annotation.Generated(value = "io.swagger.codegen.languages.SpringCodegen", date = "2019-08-17T00:22:00.237+03:00") + +public class TesInput { + @JsonProperty("name") + private String name = null; + + @JsonProperty("description") + private String description = null; + + @JsonProperty("url") + private String url = null; + + @JsonProperty("path") + private String path = null; + + @JsonProperty("type") + private TesFileType type = null; + + @JsonProperty("content") + private String content = null; + + public TesInput name(String name) { + this.name = name; + return this; + } + + /** + * Get name + * @return name + **/ + @ApiModelProperty(value = "") + + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public TesInput description(String description) { + this.description = description; + return this; + } + + /** + * Get description + * @return description + **/ + @ApiModelProperty(value = "") + + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public TesInput url(String url) { + this.url = url; + return this; + } + + /** + * REQUIRED, unless \"content\" is set. URL in long term storage, for example: s3://my-object-store/file1 gs://my-bucket/file2 file:///path/to/my/file /path/to/my/file etc... + * @return url + **/ + @ApiModelProperty(value = "REQUIRED, unless \"content\" is set. URL in long term storage, for example: s3://my-object-store/file1 gs://my-bucket/file2 file:///path/to/my/file /path/to/my/file etc...") + + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public TesInput path(String path) { + this.path = path; + return this; + } + + /** + * Path of the file inside the container. Must be an absolute path. + * @return path + **/ + @ApiModelProperty(value = "Path of the file inside the container. Must be an absolute path.") + + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public TesInput type(TesFileType type) { + this.type = type; + return this; + } + + /** + * Type of the file, FILE or DIRECTORY + * @return type + **/ + @ApiModelProperty(value = "Type of the file, FILE or DIRECTORY") + + @Valid + + public TesFileType getType() { + return type; + } + + public void setType(TesFileType type) { + this.type = type; + } + + public TesInput content(String content) { + this.content = content; + return this; + } + + /** + * File content literal. Implementations should support a minimum of 128 KiB in this field and may define its own maximum. UTF-8 encoded If content is not empty, \"url\" must be ignored. + * @return content + **/ + @ApiModelProperty(value = "File content literal. Implementations should support a minimum of 128 KiB in this field and may define its own maximum. UTF-8 encoded If content is not empty, \"url\" must be ignored.") + + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TesInput tesInput = (TesInput) o; + return Objects.equals(this.name, tesInput.name) && + Objects.equals(this.description, tesInput.description) && + Objects.equals(this.url, tesInput.url) && + Objects.equals(this.path, tesInput.path) && + Objects.equals(this.type, tesInput.type) && + Objects.equals(this.content, tesInput.content); + } + + @Override + public int hashCode() { + return Objects.hash(name, description, url, path, type, content); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class TesInput {\n"); + + sb.append(" name: ").append(toIndentedString(name)).append("\n"); + sb.append(" description: ").append(toIndentedString(description)).append("\n"); + sb.append(" url: ").append(toIndentedString(url)).append("\n"); + sb.append(" path: ").append(toIndentedString(path)).append("\n"); + sb.append(" type: ").append(toIndentedString(type)).append("\n"); + sb.append(" content: ").append(toIndentedString(content)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java new file mode 100644 index 0000000000..e219b4a446 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java @@ -0,0 +1,118 @@ +package com.epam.pipeline.tesadapter.entity; + +import java.util.Objects; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.ArrayList; +import java.util.List; +import org.springframework.validation.annotation.Validated; +import javax.validation.Valid; + +/** + * ListTasksResponse describes a response from the ListTasks endpoint. + */ +@ApiModel(description = "ListTasksResponse describes a response from the ListTasks endpoint.") +@Validated +@javax.annotation.Generated(value = "io.swagger.codegen.languages.SpringCodegen", date = "2019-08-17T00:22:00.237+03:00") + +public class TesListTasksResponse { + @JsonProperty("tasks") + @Valid + private List tasks = null; + + @JsonProperty("next_page_token") + private String nextPageToken = null; + + public TesListTasksResponse tasks(List tasks) { + this.tasks = tasks; + return this; + } + + public TesListTasksResponse addTasksItem(TesTask tasksItem) { + if (this.tasks == null) { + this.tasks = new ArrayList(); + } + this.tasks.add(tasksItem); + return this; + } + + /** + * List of tasks. + * @return tasks + **/ + @ApiModelProperty(value = "List of tasks.") + + @Valid + + public List getTasks() { + return tasks; + } + + public void setTasks(List tasks) { + this.tasks = tasks; + } + + public TesListTasksResponse nextPageToken(String nextPageToken) { + this.nextPageToken = nextPageToken; + return this; + } + + /** + * Token used to return the next page of results. See TaskListRequest.next_page_token + * @return nextPageToken + **/ + @ApiModelProperty(value = "Token used to return the next page of results. See TaskListRequest.next_page_token") + + + public String getNextPageToken() { + return nextPageToken; + } + + public void setNextPageToken(String nextPageToken) { + this.nextPageToken = nextPageToken; + } + + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TesListTasksResponse tesListTasksResponse = (TesListTasksResponse) o; + return Objects.equals(this.tasks, tesListTasksResponse.tasks) && + Objects.equals(this.nextPageToken, tesListTasksResponse.nextPageToken); + } + + @Override + public int hashCode() { + return Objects.hash(tasks, nextPageToken); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class TesListTasksResponse {\n"); + + sb.append(" tasks: ").append(toIndentedString(tasks)).append("\n"); + sb.append(" nextPageToken: ").append(toIndentedString(nextPageToken)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java new file mode 100644 index 0000000000..c1c118d11e --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java @@ -0,0 +1,181 @@ +package com.epam.pipeline.tesadapter.entity; + +import java.util.Objects; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import org.springframework.validation.annotation.Validated; +import javax.validation.Valid; + +/** + * Output describes Task output files. + */ +@ApiModel(description = "Output describes Task output files.") +@Validated +@javax.annotation.Generated(value = "io.swagger.codegen.languages.SpringCodegen", date = "2019-08-17T00:22:00.237+03:00") + +public class TesOutput { + @JsonProperty("name") + private String name = null; + + @JsonProperty("description") + private String description = null; + + @JsonProperty("url") + private String url = null; + + @JsonProperty("path") + private String path = null; + + @JsonProperty("type") + private TesFileType type = null; + + public TesOutput name(String name) { + this.name = name; + return this; + } + + /** + * Get name + * @return name + **/ + @ApiModelProperty(value = "") + + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public TesOutput description(String description) { + this.description = description; + return this; + } + + /** + * Get description + * @return description + **/ + @ApiModelProperty(value = "") + + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public TesOutput url(String url) { + this.url = url; + return this; + } + + /** + * URL in long term storage, for example: s3://my-object-store/file1 gs://my-bucket/file2 file:///path/to/my/file /path/to/my/file etc... + * @return url + **/ + @ApiModelProperty(value = "URL in long term storage, for example: s3://my-object-store/file1 gs://my-bucket/file2 file:///path/to/my/file /path/to/my/file etc...") + + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public TesOutput path(String path) { + this.path = path; + return this; + } + + /** + * Path of the file inside the container. Must be an absolute path. + * @return path + **/ + @ApiModelProperty(value = "Path of the file inside the container. Must be an absolute path.") + + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public TesOutput type(TesFileType type) { + this.type = type; + return this; + } + + /** + * Type of the file, FILE or DIRECTORY + * @return type + **/ + @ApiModelProperty(value = "Type of the file, FILE or DIRECTORY") + + @Valid + + public TesFileType getType() { + return type; + } + + public void setType(TesFileType type) { + this.type = type; + } + + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TesOutput tesOutput = (TesOutput) o; + return Objects.equals(this.name, tesOutput.name) && + Objects.equals(this.description, tesOutput.description) && + Objects.equals(this.url, tesOutput.url) && + Objects.equals(this.path, tesOutput.path) && + Objects.equals(this.type, tesOutput.type); + } + + @Override + public int hashCode() { + return Objects.hash(name, description, url, path, type); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class TesOutput {\n"); + + sb.append(" name: ").append(toIndentedString(name)).append("\n"); + sb.append(" description: ").append(toIndentedString(description)).append("\n"); + sb.append(" url: ").append(toIndentedString(url)).append("\n"); + sb.append(" path: ").append(toIndentedString(path)).append("\n"); + sb.append(" type: ").append(toIndentedString(type)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java new file mode 100644 index 0000000000..da644b5c79 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java @@ -0,0 +1,129 @@ +package com.epam.pipeline.tesadapter.entity; + +import java.util.Objects; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import org.springframework.validation.annotation.Validated; + +/** + * OutputFileLog describes a single output file. This describes file details after the task has completed successfully, for logging purposes. + */ +@ApiModel(description = "OutputFileLog describes a single output file. This describes file details after the task has completed successfully, for logging purposes.") +@Validated +@javax.annotation.Generated(value = "io.swagger.codegen.languages.SpringCodegen", date = "2019-08-17T00:22:00.237+03:00") + +public class TesOutputFileLog { + @JsonProperty("url") + private String url = null; + + @JsonProperty("path") + private String path = null; + + @JsonProperty("size_bytes") + private String sizeBytes = null; + + public TesOutputFileLog url(String url) { + this.url = url; + return this; + } + + /** + * URL of the file in storage, e.g. s3://bucket/file.txt + * @return url + **/ + @ApiModelProperty(value = "URL of the file in storage, e.g. s3://bucket/file.txt") + + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public TesOutputFileLog path(String path) { + this.path = path; + return this; + } + + /** + * Path of the file inside the container. Must be an absolute path. + * @return path + **/ + @ApiModelProperty(value = "Path of the file inside the container. Must be an absolute path.") + + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public TesOutputFileLog sizeBytes(String sizeBytes) { + this.sizeBytes = sizeBytes; + return this; + } + + /** + * Size of the file in bytes. + * @return sizeBytes + **/ + @ApiModelProperty(value = "Size of the file in bytes.") + + + public String getSizeBytes() { + return sizeBytes; + } + + public void setSizeBytes(String sizeBytes) { + this.sizeBytes = sizeBytes; + } + + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TesOutputFileLog tesOutputFileLog = (TesOutputFileLog) o; + return Objects.equals(this.url, tesOutputFileLog.url) && + Objects.equals(this.path, tesOutputFileLog.path) && + Objects.equals(this.sizeBytes, tesOutputFileLog.sizeBytes); + } + + @Override + public int hashCode() { + return Objects.hash(url, path, sizeBytes); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class TesOutputFileLog {\n"); + + sb.append(" url: ").append(toIndentedString(url)).append("\n"); + sb.append(" path: ").append(toIndentedString(path)).append("\n"); + sb.append(" sizeBytes: ").append(toIndentedString(sizeBytes)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java new file mode 100644 index 0000000000..4d17d7d6d1 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java @@ -0,0 +1,191 @@ +package com.epam.pipeline.tesadapter.entity; + +import java.util.Objects; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import java.util.ArrayList; +import java.util.List; +import org.springframework.validation.annotation.Validated; +import javax.validation.Valid; + +/** + * Resources describes the resources requested by a task. + */ +@ApiModel(description = "Resources describes the resources requested by a task.") +@Validated +@javax.annotation.Generated(value = "io.swagger.codegen.languages.SpringCodegen", date = "2019-08-17T00:22:00.237+03:00") + +public class TesResources { + @JsonProperty("cpu_cores") + private Long cpuCores = null; + + @JsonProperty("preemptible") + private Boolean preemptible = null; + + @JsonProperty("ram_gb") + private Double ramGb = null; + + @JsonProperty("disk_gb") + private Double diskGb = null; + + @JsonProperty("zones") + @Valid + private List zones = null; + + public TesResources cpuCores(Long cpuCores) { + this.cpuCores = cpuCores; + return this; + } + + /** + * Requested number of CPUs + * @return cpuCores + **/ + @ApiModelProperty(value = "Requested number of CPUs") + + + public Long getCpuCores() { + return cpuCores; + } + + public void setCpuCores(Long cpuCores) { + this.cpuCores = cpuCores; + } + + public TesResources preemptible(Boolean preemptible) { + this.preemptible = preemptible; + return this; + } + + /** + * Is the task allowed to run on preemptible compute instances (e.g. AWS Spot)? + * @return preemptible + **/ + @ApiModelProperty(value = "Is the task allowed to run on preemptible compute instances (e.g. AWS Spot)?") + + + public Boolean isPreemptible() { + return preemptible; + } + + public void setPreemptible(Boolean preemptible) { + this.preemptible = preemptible; + } + + public TesResources ramGb(Double ramGb) { + this.ramGb = ramGb; + return this; + } + + /** + * Requested RAM required in gigabytes (GB) + * @return ramGb + **/ + @ApiModelProperty(value = "Requested RAM required in gigabytes (GB)") + + + public Double getRamGb() { + return ramGb; + } + + public void setRamGb(Double ramGb) { + this.ramGb = ramGb; + } + + public TesResources diskGb(Double diskGb) { + this.diskGb = diskGb; + return this; + } + + /** + * Requested disk size in gigabytes (GB) + * @return diskGb + **/ + @ApiModelProperty(value = "Requested disk size in gigabytes (GB)") + + + public Double getDiskGb() { + return diskGb; + } + + public void setDiskGb(Double diskGb) { + this.diskGb = diskGb; + } + + public TesResources zones(List zones) { + this.zones = zones; + return this; + } + + public TesResources addZonesItem(String zonesItem) { + if (this.zones == null) { + this.zones = new ArrayList(); + } + this.zones.add(zonesItem); + return this; + } + + /** + * Request that the task be run in these compute zones. + * @return zones + **/ + @ApiModelProperty(value = "Request that the task be run in these compute zones.") + + + public List getZones() { + return zones; + } + + public void setZones(List zones) { + this.zones = zones; + } + + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TesResources tesResources = (TesResources) o; + return Objects.equals(this.cpuCores, tesResources.cpuCores) && + Objects.equals(this.preemptible, tesResources.preemptible) && + Objects.equals(this.ramGb, tesResources.ramGb) && + Objects.equals(this.diskGb, tesResources.diskGb) && + Objects.equals(this.zones, tesResources.zones); + } + + @Override + public int hashCode() { + return Objects.hash(cpuCores, preemptible, ramGb, diskGb, zones); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class TesResources {\n"); + + sb.append(" cpuCores: ").append(toIndentedString(cpuCores)).append("\n"); + sb.append(" preemptible: ").append(toIndentedString(preemptible)).append("\n"); + sb.append(" ramGb: ").append(toIndentedString(ramGb)).append("\n"); + sb.append(" diskGb: ").append(toIndentedString(diskGb)).append("\n"); + sb.append(" zones: ").append(toIndentedString(zones)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java new file mode 100644 index 0000000000..bee2c00795 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java @@ -0,0 +1,141 @@ +package com.epam.pipeline.tesadapter.entity; + +import java.util.Objects; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import java.util.ArrayList; +import java.util.List; +import org.springframework.validation.annotation.Validated; +import javax.validation.Valid; + +/** + * ServiceInfo describes information about the service, such as storage details, resource availability, and other documentation. + */ +@ApiModel(description = "ServiceInfo describes information about the service, such as storage details, resource availability, and other documentation.") +@Validated +@javax.annotation.Generated(value = "io.swagger.codegen.languages.SpringCodegen", date = "2019-08-17T00:22:00.237+03:00") + +public class TesServiceInfo { + @JsonProperty("name") + private String name = null; + + @JsonProperty("doc") + private String doc = null; + + @JsonProperty("storage") + @Valid + private List storage = null; + + public TesServiceInfo name(String name) { + this.name = name; + return this; + } + + /** + * Returns the name of the service, e.g. \"ohsu-compbio-funnel\". + * @return name + **/ + @ApiModelProperty(value = "Returns the name of the service, e.g. \"ohsu-compbio-funnel\".") + + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public TesServiceInfo doc(String doc) { + this.doc = doc; + return this; + } + + /** + * Returns a documentation string, e.g. \"Hey, we're OHSU Comp. Bio!\". + * @return doc + **/ + @ApiModelProperty(value = "Returns a documentation string, e.g. \"Hey, we're OHSU Comp. Bio!\".") + + + public String getDoc() { + return doc; + } + + public void setDoc(String doc) { + this.doc = doc; + } + + public TesServiceInfo storage(List storage) { + this.storage = storage; + return this; + } + + public TesServiceInfo addStorageItem(String storageItem) { + if (this.storage == null) { + this.storage = new ArrayList(); + } + this.storage.add(storageItem); + return this; + } + + /** + * Lists some, but not necessarily all, storage locations supported by the service. Must be in a valid URL format. e.g. file:///path/to/local/funnel-storage s3://ohsu-compbio-funnel/storage etc. + * @return storage + **/ + @ApiModelProperty(value = "Lists some, but not necessarily all, storage locations supported by the service. Must be in a valid URL format. e.g. file:///path/to/local/funnel-storage s3://ohsu-compbio-funnel/storage etc.") + + + public List getStorage() { + return storage; + } + + public void setStorage(List storage) { + this.storage = storage; + } + + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TesServiceInfo tesServiceInfo = (TesServiceInfo) o; + return Objects.equals(this.name, tesServiceInfo.name) && + Objects.equals(this.doc, tesServiceInfo.doc) && + Objects.equals(this.storage, tesServiceInfo.storage); + } + + @Override + public int hashCode() { + return Objects.hash(name, doc, storage); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class TesServiceInfo {\n"); + + sb.append(" name: ").append(toIndentedString(name)).append("\n"); + sb.append(" doc: ").append(toIndentedString(doc)).append("\n"); + sb.append(" storage: ").append(toIndentedString(storage)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java new file mode 100644 index 0000000000..7cc9eb0674 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java @@ -0,0 +1,52 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.fasterxml.jackson.annotation.JsonValue; + +import com.fasterxml.jackson.annotation.JsonCreator; + +/** + * Task states. - UNKNOWN: The state of the task is unknown. This provides a safe default for messages where this field is missing, for example, so that a missing field does not accidentally imply that the state is QUEUED. - QUEUED: The task is queued. - INITIALIZING: The task has been assigned to a worker and is currently preparing to run. For example, the worker may be turning on, downloading input files, etc. - RUNNING: The task is running. Input files are downloaded and the first Executor has been started. - PAUSED: The task is paused. An implementation may have the ability to pause a task, but this is not required. - COMPLETE: The task has completed running. Executors have exited without error and output files have been successfully uploaded. - EXECUTOR_ERROR: The task encountered an error in one of the Executor processes. Generally, this means that an Executor exited with a non-zero exit code. - SYSTEM_ERROR: The task was stopped due to a system error, but not from an Executor, for example an upload failed due to network issues, the worker's ran out of disk space, etc. - CANCELED: The task was canceled by the user. + */ +public enum TesState { + + UNKNOWN("UNKNOWN"), + + QUEUED("QUEUED"), + + INITIALIZING("INITIALIZING"), + + RUNNING("RUNNING"), + + PAUSED("PAUSED"), + + COMPLETE("COMPLETE"), + + EXECUTOR_ERROR("EXECUTOR_ERROR"), + + SYSTEM_ERROR("SYSTEM_ERROR"), + + CANCELED("CANCELED"); + + private String value; + + TesState(String value) { + this.value = value; + } + + @Override + @JsonValue + public String toString() { + return String.valueOf(value); + } + + @JsonCreator + public static TesState fromValue(String text) { + for (TesState b : TesState.values()) { + if (String.valueOf(b.value).equals(text)) { + return b; + } + } + return null; + } +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java new file mode 100644 index 0000000000..2d9ff55f5b --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java @@ -0,0 +1,420 @@ +package com.epam.pipeline.tesadapter.entity; + +import java.util.Objects; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.springframework.validation.annotation.Validated; +import javax.validation.Valid; + +/** + * Task describes an instance of a task. + */ +@ApiModel(description = "Task describes an instance of a task.") +@Validated +@javax.annotation.Generated(value = "io.swagger.codegen.languages.SpringCodegen", date = "2019-08-17T00:22:00.237+03:00") + +public class TesTask { + @JsonProperty("id") + private String id = null; + + @JsonProperty("state") + private TesState state = null; + + @JsonProperty("name") + private String name = null; + + @JsonProperty("description") + private String description = null; + + @JsonProperty("inputs") + @Valid + private List inputs = null; + + @JsonProperty("outputs") + @Valid + private List outputs = null; + + @JsonProperty("resources") + private TesResources resources = null; + + @JsonProperty("executors") + @Valid + private List executors = null; + + @JsonProperty("volumes") + @Valid + private List volumes = null; + + @JsonProperty("tags") + @Valid + private Map tags = null; + + @JsonProperty("logs") + @Valid + private List logs = null; + + @JsonProperty("creation_time") + private String creationTime = null; + + public TesTask id(String id) { + this.id = id; + return this; + } + + /** + * Task identifier assigned by the server. + * @return id + **/ + @ApiModelProperty(value = "Task identifier assigned by the server.") + + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public TesTask state(TesState state) { + this.state = state; + return this; + } + + /** + * Get state + * @return state + **/ + @ApiModelProperty(value = "") + + @Valid + + public TesState getState() { + return state; + } + + public void setState(TesState state) { + this.state = state; + } + + public TesTask name(String name) { + this.name = name; + return this; + } + + /** + * Get name + * @return name + **/ + @ApiModelProperty(value = "") + + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public TesTask description(String description) { + this.description = description; + return this; + } + + /** + * Get description + * @return description + **/ + @ApiModelProperty(value = "") + + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public TesTask inputs(List inputs) { + this.inputs = inputs; + return this; + } + + public TesTask addInputsItem(TesInput inputsItem) { + if (this.inputs == null) { + this.inputs = new ArrayList(); + } + this.inputs.add(inputsItem); + return this; + } + + /** + * Input files. Inputs will be downloaded and mounted into the executor container. + * @return inputs + **/ + @ApiModelProperty(value = "Input files. Inputs will be downloaded and mounted into the executor container.") + + @Valid + + public List getInputs() { + return inputs; + } + + public void setInputs(List inputs) { + this.inputs = inputs; + } + + public TesTask outputs(List outputs) { + this.outputs = outputs; + return this; + } + + public TesTask addOutputsItem(TesOutput outputsItem) { + if (this.outputs == null) { + this.outputs = new ArrayList(); + } + this.outputs.add(outputsItem); + return this; + } + + /** + * Output files. Outputs will be uploaded from the executor container to long-term storage. + * @return outputs + **/ + @ApiModelProperty(value = "Output files. Outputs will be uploaded from the executor container to long-term storage.") + + @Valid + + public List getOutputs() { + return outputs; + } + + public void setOutputs(List outputs) { + this.outputs = outputs; + } + + public TesTask resources(TesResources resources) { + this.resources = resources; + return this; + } + + /** + * Request that the task be run with these resources. + * @return resources + **/ + @ApiModelProperty(value = "Request that the task be run with these resources.") + + @Valid + + public TesResources getResources() { + return resources; + } + + public void setResources(TesResources resources) { + this.resources = resources; + } + + public TesTask executors(List executors) { + this.executors = executors; + return this; + } + + public TesTask addExecutorsItem(TesExecutor executorsItem) { + if (this.executors == null) { + this.executors = new ArrayList(); + } + this.executors.add(executorsItem); + return this; + } + + /** + * A list of executors to be run, sequentially. Execution stops on the first error. + * @return executors + **/ + @ApiModelProperty(value = "A list of executors to be run, sequentially. Execution stops on the first error.") + + @Valid + + public List getExecutors() { + return executors; + } + + public void setExecutors(List executors) { + this.executors = executors; + } + + public TesTask volumes(List volumes) { + this.volumes = volumes; + return this; + } + + public TesTask addVolumesItem(String volumesItem) { + if (this.volumes == null) { + this.volumes = new ArrayList(); + } + this.volumes.add(volumesItem); + return this; + } + + /** + * Volumes are directories which may be used to share data between Executors. Volumes are initialized as empty directories by the system when the task starts and are mounted at the same path in each Executor. For example, given a volume defined at \"/vol/A\", executor 1 may write a file to \"/vol/A/exec1.out.txt\", then executor 2 may read from that file. (Essentially, this translates to a `docker run -v` flag where the container path is the same for each executor). + * @return volumes + **/ + @ApiModelProperty(value = "Volumes are directories which may be used to share data between Executors. Volumes are initialized as empty directories by the system when the task starts and are mounted at the same path in each Executor. For example, given a volume defined at \"/vol/A\", executor 1 may write a file to \"/vol/A/exec1.out.txt\", then executor 2 may read from that file. (Essentially, this translates to a `docker run -v` flag where the container path is the same for each executor).") + + + public List getVolumes() { + return volumes; + } + + public void setVolumes(List volumes) { + this.volumes = volumes; + } + + public TesTask tags(Map tags) { + this.tags = tags; + return this; + } + + public TesTask putTagsItem(String key, String tagsItem) { + if (this.tags == null) { + this.tags = new HashMap(); + } + this.tags.put(key, tagsItem); + return this; + } + + /** + * A key-value map of arbitrary tags. + * @return tags + **/ + @ApiModelProperty(value = "A key-value map of arbitrary tags.") + + + public Map getTags() { + return tags; + } + + public void setTags(Map tags) { + this.tags = tags; + } + + public TesTask logs(List logs) { + this.logs = logs; + return this; + } + + public TesTask addLogsItem(TesTaskLog logsItem) { + if (this.logs == null) { + this.logs = new ArrayList(); + } + this.logs.add(logsItem); + return this; + } + + /** + * Task logging information. Normally, this will contain only one entry, but in the case where a task fails and is retried, an entry will be appended to this list. + * @return logs + **/ + @ApiModelProperty(value = "Task logging information. Normally, this will contain only one entry, but in the case where a task fails and is retried, an entry will be appended to this list.") + + @Valid + + public List getLogs() { + return logs; + } + + public void setLogs(List logs) { + this.logs = logs; + } + + public TesTask creationTime(String creationTime) { + this.creationTime = creationTime; + return this; + } + + /** + * Date + time the task was created, in RFC 3339 format. This is set by the system, not the client. + * @return creationTime + **/ + @ApiModelProperty(value = "Date + time the task was created, in RFC 3339 format. This is set by the system, not the client.") + + + public String getCreationTime() { + return creationTime; + } + + public void setCreationTime(String creationTime) { + this.creationTime = creationTime; + } + + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TesTask tesTask = (TesTask) o; + return Objects.equals(this.id, tesTask.id) && + Objects.equals(this.state, tesTask.state) && + Objects.equals(this.name, tesTask.name) && + Objects.equals(this.description, tesTask.description) && + Objects.equals(this.inputs, tesTask.inputs) && + Objects.equals(this.outputs, tesTask.outputs) && + Objects.equals(this.resources, tesTask.resources) && + Objects.equals(this.executors, tesTask.executors) && + Objects.equals(this.volumes, tesTask.volumes) && + Objects.equals(this.tags, tesTask.tags) && + Objects.equals(this.logs, tesTask.logs) && + Objects.equals(this.creationTime, tesTask.creationTime); + } + + @Override + public int hashCode() { + return Objects.hash(id, state, name, description, inputs, outputs, resources, executors, volumes, tags, logs, creationTime); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class TesTask {\n"); + + sb.append(" id: ").append(toIndentedString(id)).append("\n"); + sb.append(" state: ").append(toIndentedString(state)).append("\n"); + sb.append(" name: ").append(toIndentedString(name)).append("\n"); + sb.append(" description: ").append(toIndentedString(description)).append("\n"); + sb.append(" inputs: ").append(toIndentedString(inputs)).append("\n"); + sb.append(" outputs: ").append(toIndentedString(outputs)).append("\n"); + sb.append(" resources: ").append(toIndentedString(resources)).append("\n"); + sb.append(" executors: ").append(toIndentedString(executors)).append("\n"); + sb.append(" volumes: ").append(toIndentedString(volumes)).append("\n"); + sb.append(" tags: ").append(toIndentedString(tags)).append("\n"); + sb.append(" logs: ").append(toIndentedString(logs)).append("\n"); + sb.append(" creationTime: ").append(toIndentedString(creationTime)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java new file mode 100644 index 0000000000..1cd56be13e --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java @@ -0,0 +1,248 @@ +package com.epam.pipeline.tesadapter.entity; + +import java.util.Objects; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.springframework.validation.annotation.Validated; +import javax.validation.Valid; + +/** + * TaskLog describes logging information related to a Task. + */ +@ApiModel(description = "TaskLog describes logging information related to a Task.") +@Validated +@javax.annotation.Generated(value = "io.swagger.codegen.languages.SpringCodegen", date = "2019-08-17T00:22:00.237+03:00") + +public class TesTaskLog { + @JsonProperty("logs") + @Valid + private List logs = null; + + @JsonProperty("metadata") + @Valid + private Map metadata = null; + + @JsonProperty("start_time") + private String startTime = null; + + @JsonProperty("end_time") + private String endTime = null; + + @JsonProperty("outputs") + @Valid + private List outputs = null; + + @JsonProperty("system_logs") + @Valid + private List systemLogs = null; + + public TesTaskLog logs(List logs) { + this.logs = logs; + return this; + } + + public TesTaskLog addLogsItem(TesExecutorLog logsItem) { + if (this.logs == null) { + this.logs = new ArrayList(); + } + this.logs.add(logsItem); + return this; + } + + /** + * Logs for each executor + * @return logs + **/ + @ApiModelProperty(value = "Logs for each executor") + + @Valid + + public List getLogs() { + return logs; + } + + public void setLogs(List logs) { + this.logs = logs; + } + + public TesTaskLog metadata(Map metadata) { + this.metadata = metadata; + return this; + } + + public TesTaskLog putMetadataItem(String key, String metadataItem) { + if (this.metadata == null) { + this.metadata = new HashMap(); + } + this.metadata.put(key, metadataItem); + return this; + } + + /** + * Arbitrary logging metadata included by the implementation. + * @return metadata + **/ + @ApiModelProperty(value = "Arbitrary logging metadata included by the implementation.") + + + public Map getMetadata() { + return metadata; + } + + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + + public TesTaskLog startTime(String startTime) { + this.startTime = startTime; + return this; + } + + /** + * When the task started, in RFC 3339 format. + * @return startTime + **/ + @ApiModelProperty(value = "When the task started, in RFC 3339 format.") + + + public String getStartTime() { + return startTime; + } + + public void setStartTime(String startTime) { + this.startTime = startTime; + } + + public TesTaskLog endTime(String endTime) { + this.endTime = endTime; + return this; + } + + /** + * When the task ended, in RFC 3339 format. + * @return endTime + **/ + @ApiModelProperty(value = "When the task ended, in RFC 3339 format.") + + + public String getEndTime() { + return endTime; + } + + public void setEndTime(String endTime) { + this.endTime = endTime; + } + + public TesTaskLog outputs(List outputs) { + this.outputs = outputs; + return this; + } + + public TesTaskLog addOutputsItem(TesOutputFileLog outputsItem) { + if (this.outputs == null) { + this.outputs = new ArrayList(); + } + this.outputs.add(outputsItem); + return this; + } + + /** + * Information about all output files. Directory outputs are flattened into separate items. + * @return outputs + **/ + @ApiModelProperty(value = "Information about all output files. Directory outputs are flattened into separate items.") + + @Valid + + public List getOutputs() { + return outputs; + } + + public void setOutputs(List outputs) { + this.outputs = outputs; + } + + public TesTaskLog systemLogs(List systemLogs) { + this.systemLogs = systemLogs; + return this; + } + + public TesTaskLog addSystemLogsItem(String systemLogsItem) { + if (this.systemLogs == null) { + this.systemLogs = new ArrayList(); + } + this.systemLogs.add(systemLogsItem); + return this; + } + + /** + * System logs are any logs the system decides are relevant, which are not tied directly to an Executor process. Content is implementation specific: format, size, etc. System logs may be collected here to provide convenient access. For example, the system may include the name of the host where the task is executing, an error message that caused a SYSTEM_ERROR state (e.g. disk is full), etc. System logs are only included in the FULL task view. + * @return systemLogs + **/ + @ApiModelProperty(value = "System logs are any logs the system decides are relevant, which are not tied directly to an Executor process. Content is implementation specific: format, size, etc. System logs may be collected here to provide convenient access. For example, the system may include the name of the host where the task is executing, an error message that caused a SYSTEM_ERROR state (e.g. disk is full), etc. System logs are only included in the FULL task view.") + + + public List getSystemLogs() { + return systemLogs; + } + + public void setSystemLogs(List systemLogs) { + this.systemLogs = systemLogs; + } + + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TesTaskLog tesTaskLog = (TesTaskLog) o; + return Objects.equals(this.logs, tesTaskLog.logs) && + Objects.equals(this.metadata, tesTaskLog.metadata) && + Objects.equals(this.startTime, tesTaskLog.startTime) && + Objects.equals(this.endTime, tesTaskLog.endTime) && + Objects.equals(this.outputs, tesTaskLog.outputs) && + Objects.equals(this.systemLogs, tesTaskLog.systemLogs); + } + + @Override + public int hashCode() { + return Objects.hash(logs, metadata, startTime, endTime, outputs, systemLogs); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class TesTaskLog {\n"); + + sb.append(" logs: ").append(toIndentedString(logs)).append("\n"); + sb.append(" metadata: ").append(toIndentedString(metadata)).append("\n"); + sb.append(" startTime: ").append(toIndentedString(startTime)).append("\n"); + sb.append(" endTime: ").append(toIndentedString(endTime)).append("\n"); + sb.append(" outputs: ").append(toIndentedString(outputs)).append("\n"); + sb.append(" systemLogs: ").append(toIndentedString(systemLogs)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} + From 139c31b65de6e3a6bd83c46d95901bc3a09ca5ca Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Sun, 18 Aug 2019 00:10:48 +0300 Subject: [PATCH 011/194] added stubbed methods submit/cancel/get-task. Reformatting entities imports. added TES-API interface. --- .../tesadapter/TesAdapterApplication.java | 3 +- .../controller/TesAdapterController.java | 19 --------- .../tesadapter/controller/TesApi.java | 19 +++++++++ .../controller/TesApiController.java | 30 ++++++++++++++ .../entity/TesCancelTaskResponse.java | 5 +-- .../entity/TesCreateTaskResponse.java | 3 +- .../tesadapter/entity/TesExecutor.java | 7 +--- .../tesadapter/entity/TesExecutorLog.java | 3 +- .../tesadapter/entity/TesFileType.java | 3 +- .../pipeline/tesadapter/entity/TesInput.java | 3 +- .../entity/TesListTasksResponse.java | 6 +-- .../pipeline/tesadapter/entity/TesOutput.java | 3 +- .../tesadapter/entity/TesOutputFileLog.java | 3 +- .../tesadapter/entity/TesResources.java | 7 ++-- .../tesadapter/entity/TesServiceInfo.java | 7 ++-- .../pipeline/tesadapter/entity/TesState.java | 3 +- .../pipeline/tesadapter/entity/TesTask.java | 8 +--- .../tesadapter/entity/TesTaskLog.java | 8 +--- .../tesadapter/service/TesTaskService.java | 41 +++++++++++++++++++ 19 files changed, 123 insertions(+), 58 deletions(-) delete mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesApi.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesApiController.java diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java index 9a04a3c380..0fbd8e8490 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java @@ -8,7 +8,8 @@ @SpringBootApplication @Import({TesSwaggerConfig.class}) -@ComponentScan({"com.epam.pipeline.tesadapter.controller", "com.epam.pipeline.tesadapter.service"}) +@ComponentScan({"com.epam.pipeline.tesadapter.controller", "com.epam.pipeline.tesadapter.service", + "com.epam.pipeline.tesadapter.entity"}) public class TesAdapterApplication { public static void main(String[] args) { SpringApplication.run(TesAdapterApplication.class, args); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java deleted file mode 100644 index ce42a18220..0000000000 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.epam.pipeline.tesadapter.controller; - -import com.epam.pipeline.tesadapter.service.TesTaskService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RestController; - -@RestController -public class TesAdapterController { - @Autowired - private TesTaskService tesTaskService; - - @GetMapping("/v1/tasks/service-info") - public ResponseEntity serviceInfo() { - return new ResponseEntity<>("OK", HttpStatus.OK); - } -} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesApi.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesApi.java new file mode 100644 index 0000000000..40ffd52e90 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesApi.java @@ -0,0 +1,19 @@ +package com.epam.pipeline.tesadapter.controller; + +import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; +import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; +import com.epam.pipeline.tesadapter.entity.TesTask; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +public interface TesApi { + + @PostMapping("/v1/tasks") + @ResponseBody ResponseEntity submitTesTask(@RequestBody TesTask body); + + @GetMapping("/v1/tasks/{id}") + @ResponseBody ResponseEntity getTesTask(@PathVariable("id") String id); + + @PostMapping("/v1/tasks/{id}:cancel") + @ResponseBody ResponseEntity cancelTesTask(@PathVariable("id") String id); +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesApiController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesApiController.java new file mode 100644 index 0000000000..16d7b999e1 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesApiController.java @@ -0,0 +1,30 @@ +package com.epam.pipeline.tesadapter.controller; + +import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; +import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; +import com.epam.pipeline.tesadapter.entity.TesTask; +import com.epam.pipeline.tesadapter.service.TesTaskService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class TesApiController implements TesApi{ + @Autowired + private TesTaskService tesTaskService; + + @Override + public ResponseEntity submitTesTask(TesTask body) { + return tesTaskService.submitTesTask(new TesTask()); + } + + @Override + public ResponseEntity getTesTask(String id) { + return tesTaskService.getTesTask("STUBBED"); + } + + @Override + public ResponseEntity cancelTesTask(String id) { + return tesTaskService.cancelTesTask("STUBBED"); + } +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java index 3ae223da8d..557f3f5f37 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java @@ -1,10 +1,9 @@ package com.epam.pipeline.tesadapter.entity; -import java.util.Objects; import io.swagger.annotations.ApiModel; import org.springframework.validation.annotation.Validated; -import javax.validation.Valid; -import javax.validation.constraints.*; + +import java.util.Objects; /** * CancelTaskResponse describes a response from the CancelTask endpoint. diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java index 28baf0a1f5..02ec214bc4 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java @@ -1,11 +1,12 @@ package com.epam.pipeline.tesadapter.entity; -import java.util.Objects; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import org.springframework.validation.annotation.Validated; +import java.util.Objects; + /** * CreateTaskResponse describes a response from the CreateTask endpoint. */ diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java index f016f3eaed..ec8cb1d63f 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java @@ -1,15 +1,12 @@ package com.epam.pipeline.tesadapter.entity; -import java.util.Objects; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import org.springframework.validation.annotation.Validated; + import javax.validation.Valid; +import java.util.*; /** * Executor describes a command to be executed, and its environment. diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java index 4e9182805b..376dda1a4a 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java @@ -1,11 +1,12 @@ package com.epam.pipeline.tesadapter.entity; -import java.util.Objects; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import org.springframework.validation.annotation.Validated; +import java.util.Objects; + /** * ExecutorLog describes logging information related to an Executor. */ diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java index 5af01a74bd..2eab6fc7dc 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java @@ -1,8 +1,7 @@ package com.epam.pipeline.tesadapter.entity; -import com.fasterxml.jackson.annotation.JsonValue; - import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; /** * Gets or Sets tesFileType diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java index f683048e72..9072590a0f 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java @@ -1,11 +1,12 @@ package com.epam.pipeline.tesadapter.entity; -import java.util.Objects; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import org.springframework.validation.annotation.Validated; + import javax.validation.Valid; +import java.util.Objects; /** * Input describes Task input files. diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java index e219b4a446..021096532b 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java @@ -1,14 +1,14 @@ package com.epam.pipeline.tesadapter.entity; -import java.util.Objects; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import org.springframework.validation.annotation.Validated; +import javax.validation.Valid; import java.util.ArrayList; import java.util.List; -import org.springframework.validation.annotation.Validated; -import javax.validation.Valid; +import java.util.Objects; /** * ListTasksResponse describes a response from the ListTasks endpoint. diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java index c1c118d11e..29d4a9f23a 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java @@ -1,11 +1,12 @@ package com.epam.pipeline.tesadapter.entity; -import java.util.Objects; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import org.springframework.validation.annotation.Validated; + import javax.validation.Valid; +import java.util.Objects; /** * Output describes Task output files. diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java index da644b5c79..2807d9952f 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java @@ -1,11 +1,12 @@ package com.epam.pipeline.tesadapter.entity; -import java.util.Objects; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import org.springframework.validation.annotation.Validated; +import java.util.Objects; + /** * OutputFileLog describes a single output file. This describes file details after the task has completed successfully, for logging purposes. */ diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java index 4d17d7d6d1..64337ebed5 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java @@ -1,13 +1,14 @@ package com.epam.pipeline.tesadapter.entity; -import java.util.Objects; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import java.util.ArrayList; -import java.util.List; import org.springframework.validation.annotation.Validated; + import javax.validation.Valid; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; /** * Resources describes the resources requested by a task. diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java index bee2c00795..e15582548d 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java @@ -1,13 +1,14 @@ package com.epam.pipeline.tesadapter.entity; -import java.util.Objects; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import java.util.ArrayList; -import java.util.List; import org.springframework.validation.annotation.Validated; + import javax.validation.Valid; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; /** * ServiceInfo describes information about the service, such as storage details, resource availability, and other documentation. diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java index 7cc9eb0674..66dfa76d09 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java @@ -1,8 +1,7 @@ package com.epam.pipeline.tesadapter.entity; -import com.fasterxml.jackson.annotation.JsonValue; - import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; /** * Task states. - UNKNOWN: The state of the task is unknown. This provides a safe default for messages where this field is missing, for example, so that a missing field does not accidentally imply that the state is QUEUED. - QUEUED: The task is queued. - INITIALIZING: The task has been assigned to a worker and is currently preparing to run. For example, the worker may be turning on, downloading input files, etc. - RUNNING: The task is running. Input files are downloaded and the first Executor has been started. - PAUSED: The task is paused. An implementation may have the ability to pause a task, but this is not required. - COMPLETE: The task has completed running. Executors have exited without error and output files have been successfully uploaded. - EXECUTOR_ERROR: The task encountered an error in one of the Executor processes. Generally, this means that an Executor exited with a non-zero exit code. - SYSTEM_ERROR: The task was stopped due to a system error, but not from an Executor, for example an upload failed due to network issues, the worker's ran out of disk space, etc. - CANCELED: The task was canceled by the user. diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java index 2d9ff55f5b..e89a82c133 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java @@ -1,16 +1,12 @@ package com.epam.pipeline.tesadapter.entity; -import java.util.Objects; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import org.springframework.validation.annotation.Validated; + import javax.validation.Valid; +import java.util.*; /** * Task describes an instance of a task. diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java index 1cd56be13e..87d66214ad 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java @@ -1,16 +1,12 @@ package com.epam.pipeline.tesadapter.entity; -import java.util.Objects; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import org.springframework.validation.annotation.Validated; + import javax.validation.Valid; +import java.util.*; /** * TaskLog describes logging information related to a Task. diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index 2029596802..00254c4228 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -1,7 +1,48 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; +import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; +import com.epam.pipeline.tesadapter.entity.TesTask; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; +import java.io.IOException; + @Service public class TesTaskService { + @Autowired + private ObjectMapper objectMapper; + + public ResponseEntity submitTesTask(TesTask body) { + try { + return new ResponseEntity(objectMapper.readValue("STUBBED", + TesCreateTaskResponse.class), HttpStatus.NOT_IMPLEMENTED); + } catch (IOException e) { + e.printStackTrace(); + return new ResponseEntity(HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + public ResponseEntity getTesTask(String id) { + try { + return new ResponseEntity(objectMapper.readValue("STUBBED", TesTask.class), + HttpStatus.NOT_IMPLEMENTED); + } catch (IOException e) { + e.printStackTrace(); + return new ResponseEntity(HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + public ResponseEntity cancelTesTask(String id) { + try { + return new ResponseEntity(objectMapper.readValue("", + TesCancelTaskResponse.class), HttpStatus.NOT_IMPLEMENTED); + } catch (IOException e) { + e.printStackTrace(); + return new ResponseEntity(HttpStatus.INTERNAL_SERVER_ERROR); + } + } } From 5329d60017672c22897b4983f2cf3d2e0858678f Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Tue, 20 Aug 2019 17:41:09 +0300 Subject: [PATCH 012/194] Added task entities to tes-adapter (#8) * simple test tasks service implemented. injected to TesAdapterController and also in ComponentScan * implemented task entities with swagger-codegen and task_execution.swagger.json * update entities imports, added @Data annotation. Erased equals(), hashCode() and toString(), methods, and also Getters and Setters. Relocated @ApiModelProperty -s to proper places. * updated entities implementation. Removed unused methods. * updated entities implementation. Removed constructors from enum-classes. Added @AllArgsConstructor and @ToString annotations for them. --- .../tesadapter/TesAdapterApplication.java | 2 + .../controller/TesAdapterController.java | 5 ++ .../entity/TesCancelTaskResponse.java | 11 +++ .../entity/TesCreateTaskResponse.java | 16 ++++ .../tesadapter/entity/TesExecutor.java | 53 ++++++++++++ .../tesadapter/entity/TesExecutorLog.java | 36 +++++++++ .../tesadapter/entity/TesFileType.java | 21 +++++ .../pipeline/tesadapter/entity/TesInput.java | 43 ++++++++++ .../entity/TesListTasksResponse.java | 25 ++++++ .../pipeline/tesadapter/entity/TesOutput.java | 36 +++++++++ .../tesadapter/entity/TesOutputFileLog.java | 25 ++++++ .../tesadapter/entity/TesResources.java | 37 +++++++++ .../tesadapter/entity/TesServiceInfo.java | 32 ++++++++ .../pipeline/tesadapter/entity/TesState.java | 27 +++++++ .../pipeline/tesadapter/entity/TesTask.java | 80 +++++++++++++++++++ .../tesadapter/entity/TesTaskLog.java | 50 ++++++++++++ .../tesadapter/service/TesTaskService.java | 7 ++ 17 files changed, 506 insertions(+) create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java index 43925d2d73..9a04a3c380 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java @@ -3,10 +3,12 @@ import com.epam.pipeline.tesadapter.configuration.TesSwaggerConfig; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Import; @SpringBootApplication @Import({TesSwaggerConfig.class}) +@ComponentScan({"com.epam.pipeline.tesadapter.controller", "com.epam.pipeline.tesadapter.service"}) public class TesAdapterApplication { public static void main(String[] args) { SpringApplication.run(TesAdapterApplication.class, args); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index be83a8b6ec..ce42a18220 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -1,5 +1,7 @@ package com.epam.pipeline.tesadapter.controller; +import com.epam.pipeline.tesadapter.service.TesTaskService; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; @@ -7,6 +9,9 @@ @RestController public class TesAdapterController { + @Autowired + private TesTaskService tesTaskService; + @GetMapping("/v1/tasks/service-info") public ResponseEntity serviceInfo() { return new ResponseEntity<>("OK", HttpStatus.OK); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java new file mode 100644 index 0000000000..47a1f4fd18 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java @@ -0,0 +1,11 @@ +package com.epam.pipeline.tesadapter.entity; + +import io.swagger.annotations.ApiModel; +import lombok.Data; + + +@ApiModel(description = "CancelTaskResponse describes a response from the CancelTask endpoint.") +@Data +public class TesCancelTaskResponse { +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java new file mode 100644 index 0000000000..a558978a78 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java @@ -0,0 +1,16 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + + +@ApiModel(description = "CreateTaskResponse describes a response from the CreateTask endpoint.") +@Data +public class TesCreateTaskResponse { + @ApiModelProperty(value = "Task identifier assigned by the server.") + @JsonProperty("id") + private String id = null; +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java new file mode 100644 index 0000000000..4199448ae0 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java @@ -0,0 +1,53 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.Valid; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Data +@ApiModel(description = "Executor describes a command to be executed, and its environment.") +public class TesExecutor { + @ApiModelProperty(value = "Name of the container image, for example: " + + "ubuntu quay.io/aptible/ubuntu gcr.io/my-org/my-image etc...") + @JsonProperty("image") + private String image = null; + + @ApiModelProperty(value = "A sequence of program arguments to execute, " + + "where the first argument is the program to execute (i.e. argv).") + @JsonProperty("command") + @Valid + private List command = null; + + @ApiModelProperty(value = "The working directory that the command will be " + + "executed in. Defaults to the directory set by the container image.") + @JsonProperty("workdir") + private String workdir = null; + + @ApiModelProperty(value = "Path inside the container to a file which will " + + "be piped to the executor's stdin. Must be an absolute path.") + @JsonProperty("stdin") + private String stdin = null; + + @ApiModelProperty(value = "Path inside the container to a file where the " + + "executor's stdout will be written to. Must be an absolute path.") + @JsonProperty("stdout") + private String stdout = null; + + @ApiModelProperty(value = "Path inside the container to a file where the " + + "executor's stderr will be written to. Must be an absolute path.") + @JsonProperty("stderr") + private String stderr = null; + + @ApiModelProperty(value = "Enviromental variables to set within the container.") + @JsonProperty("env") + @Valid + private Map env = null; +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java new file mode 100644 index 0000000000..cdefa47b03 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java @@ -0,0 +1,36 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + + +@ApiModel(description = "ExecutorLog describes logging information related to an Executor.") +public class TesExecutorLog { + @ApiModelProperty(value = "Time the executor started, in RFC 3339 format.") + @JsonProperty("start_time") + private String startTime = null; + + @ApiModelProperty(value = "Time the executor ended, in RFC 3339 format.") + @JsonProperty("end_time") + private String endTime = null; + + @ApiModelProperty(value = "Stdout content. This is meant for convenience. No guarantees are made about " + + "the content. Implementations may chose different approaches: only the head, only the tail, a URL " + + "reference only, etc. In order to capture the full stdout users should set Executor.stdout to a " + + "container file path, and use Task.outputs to upload that file to permanent storage.") + @JsonProperty("stdout") + private String stdout = null; + + @ApiModelProperty(value = "Stderr content. This is meant for convenience. No guarantees are made about " + + "the content. Implementations may chose different approaches: only the head, only the tail, a URL " + + "reference only, etc. In order to capture the full stderr users should set Executor.stderr to a " + + "container file path, and use Task.outputs to upload that file to permanent storage.") + @JsonProperty("stderr") + private String stderr = null; + + @ApiModelProperty(value = "Exit code.") + @JsonProperty("exit_code") + private Integer exitCode = null; +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java new file mode 100644 index 0000000000..bc6bb54a98 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java @@ -0,0 +1,21 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.fasterxml.jackson.annotation.JsonValue; + +import com.fasterxml.jackson.annotation.JsonCreator; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.ToString; + +/** + * Gets or Sets tesFileType + */ +@AllArgsConstructor +@ToString +public enum TesFileType { + FILE("FILE"), + DIRECTORY("DIRECTORY"); + + private String value; +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java new file mode 100644 index 0000000000..14759928ef --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java @@ -0,0 +1,43 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.Valid; + + +@ApiModel(description = "Input describes Task input files.") +@Data +public class TesInput { + + @ApiModelProperty(value = "") + @JsonProperty("name") + private String name = null; + + @ApiModelProperty(value = "") + @JsonProperty("description") + private String description = null; + + @ApiModelProperty(value = "REQUIRED, unless \"content\" is set. URL in long term storage, for example: " + + "s3://my-object-store/file1 gs://my-bucket/file2 file:///path/to/my/file /path/to/my/file etc...") + @JsonProperty("url") + private String url = null; + + @ApiModelProperty(value = "Path of the file inside the container. Must be an absolute path.") + @JsonProperty("path") + private String path = null; + + @ApiModelProperty(value = "Type of the file, FILE or DIRECTORY") + @Valid + @JsonProperty("type") + private TesFileType type = null; + + @ApiModelProperty(value = "File content literal. Implementations should support a minimum of 128 KiB " + + "in this field and may define its own maximum. UTF-8 encoded If content is not empty, \"url\" " + + "must be ignored.") + @JsonProperty("content") + private String content = null; +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java new file mode 100644 index 0000000000..60775dd341 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java @@ -0,0 +1,25 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.Valid; +import java.util.ArrayList; +import java.util.List; + + +@ApiModel(description = "ListTasksResponse describes a response from the ListTasks endpoint.") +@Data +public class TesListTasksResponse { + @ApiModelProperty(value = "List of tasks.") + @JsonProperty("tasks") + @Valid + private List tasks = null; + + @ApiModelProperty(value = "Token used to return the next page of results. See TaskListRequest.next_page_token") + @JsonProperty("next_page_token") + private String nextPageToken = null; +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java new file mode 100644 index 0000000000..c96ff556ea --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java @@ -0,0 +1,36 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.Valid; + + +@ApiModel(description = "Output describes Task output files.") +@Data +public class TesOutput { + @ApiModelProperty(value = "") + @JsonProperty("name") + private String name = null; + + @ApiModelProperty(value = "") + @JsonProperty("description") + private String description = null; + + @ApiModelProperty(value = "URL in long term storage, for example: s3://my-object-store/file1 " + + "gs://my-bucket/file2 file:///path/to/my/file /path/to/my/file etc...") + @JsonProperty("url") + private String url = null; + + @ApiModelProperty(value = "Path of the file inside the container. Must be an absolute path.") + @JsonProperty("path") + private String path = null; + + @ApiModelProperty(value = "Type of the file, FILE or DIRECTORY") + @JsonProperty("type") + @Valid + private TesFileType type = null; +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java new file mode 100644 index 0000000000..f9631cde21 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java @@ -0,0 +1,25 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + + +@ApiModel(description = "OutputFileLog describes a single output file. This describes file details " + + "after the task has completed successfully, for logging purposes.") +@Data +public class TesOutputFileLog { + @ApiModelProperty(value = "URL of the file in storage, e.g. s3://bucket/file.txt") + @JsonProperty("url") + private String url = null; + + @ApiModelProperty(value = "Path of the file inside the container. Must be an absolute path.") + @JsonProperty("path") + private String path = null; + + @ApiModelProperty(value = "Size of the file in bytes.") + @JsonProperty("size_bytes") + private String sizeBytes = null; +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java new file mode 100644 index 0000000000..b07995cdec --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java @@ -0,0 +1,37 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.Valid; +import java.util.ArrayList; +import java.util.List; + + +@ApiModel(description = "Resources describes the resources requested by a task.") +@Data +public class TesResources { + @ApiModelProperty(value = "Requested number of CPUs") + @JsonProperty("cpu_cores") + private Long cpuCores = null; + + @ApiModelProperty(value = "Is the task allowed to run on preemptible compute instances (e.g. AWS Spot)?") + @JsonProperty("preemptible") + private Boolean preemptible = null; + + @ApiModelProperty(value = "Requested RAM required in gigabytes (GB)") + @JsonProperty("ram_gb") + private Double ramGb = null; + + @ApiModelProperty(value = "Requested disk size in gigabytes (GB)") + @JsonProperty("disk_gb") + private Double diskGb = null; + + @ApiModelProperty(value = "Request that the task be run in these compute zones.") + @JsonProperty("zones") + @Valid + private List zones = null; +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java new file mode 100644 index 0000000000..1a6510b9be --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java @@ -0,0 +1,32 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.Valid; +import java.util.ArrayList; +import java.util.List; + + +@ApiModel(description = "ServiceInfo describes information about the service, such as storage details, \r" + + "esource availability, and other documentation.") +@Data +public class TesServiceInfo { + @ApiModelProperty(value = "Returns the name of the service, e.g. \"ohsu-compbio-funnel\".") + @JsonProperty("name") + private String name = null; + + @ApiModelProperty(value = "Returns a documentation string, e.g. \"Hey, we're OHSU Comp. Bio!\".") + @JsonProperty("doc") + private String doc = null; + + @ApiModelProperty(value = "Lists some, but not necessarily all, storage locations supported by the service. " + + " Must be in a valid URL format. e.g. file:///path/to/local/funnel-storage " + + "s3://ohsu-compbio-funnel/storage etc.") + @JsonProperty("storage") + @Valid + private List storage = null; +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java new file mode 100644 index 0000000000..697eccca14 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java @@ -0,0 +1,27 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; +import lombok.AllArgsConstructor; +import lombok.ToString; + +/** + * Task states. - UNKNOWN: The state of the task is unknown. This provides a safe default for messages where this field is missing, for example, so that a missing field does not accidentally imply that the state is QUEUED. - QUEUED: The task is queued. - INITIALIZING: The task has been assigned to a worker and is currently preparing to run. For example, the worker may be turning on, downloading input files, etc. - RUNNING: The task is running. Input files are downloaded and the first Executor has been started. - PAUSED: The task is paused. An implementation may have the ability to pause a task, but this is not required. - COMPLETE: The task has completed running. Executors have exited without error and output files have been successfully uploaded. - EXECUTOR_ERROR: The task encountered an error in one of the Executor processes. Generally, this means that an Executor exited with a non-zero exit code. - SYSTEM_ERROR: The task was stopped due to a system error, but not from an Executor, for example an upload failed due to network issues, the worker's ran out of disk space, etc. - CANCELED: The task was canceled by the user. + */ +@AllArgsConstructor +@ToString +public enum TesState { + + UNKNOWN("UNKNOWN"), + QUEUED("QUEUED"), + INITIALIZING("INITIALIZING"), + RUNNING("RUNNING"), + PAUSED("PAUSED"), + COMPLETE("COMPLETE"), + EXECUTOR_ERROR("EXECUTOR_ERROR"), + SYSTEM_ERROR("SYSTEM_ERROR"), + CANCELED("CANCELED"); + + private String value; +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java new file mode 100644 index 0000000000..8a8c016f7b --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java @@ -0,0 +1,80 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.Valid; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +@ApiModel(description = "Task describes an instance of a task.") +@Data +public class TesTask { + @ApiModelProperty(value = "Task identifier assigned by the server.") + @JsonProperty("id") + private String id = null; + + @ApiModelProperty(value = "") + @Valid + @JsonProperty("state") + private TesState state = null; + + @ApiModelProperty(value = "") + @JsonProperty("name") + private String name = null; + + @ApiModelProperty(value = "") + @JsonProperty("description") + private String description = null; + + @ApiModelProperty(value = "Input files. Inputs will be downloaded and mounted into the executor container.") + @JsonProperty("inputs") + @Valid + private List inputs = null; + + @ApiModelProperty(value = "Output files. Outputs will be uploaded from the executor container to long-term storage.") + @JsonProperty("outputs") + @Valid + private List outputs = null; + + @ApiModelProperty(value = "Request that the task be run with these resources.") + @JsonProperty("resources") + @Valid + private TesResources resources = null; + + @ApiModelProperty(value = "A list of executors to be run, sequentially. Execution stops on the first error.") + @JsonProperty("executors") + @Valid + private List executors = null; + + @ApiModelProperty(value = "Volumes are directories which may be used to share data between Executors. " + + "Volumes are initialized as empty directories by the system when the task starts and are mounted " + + "at the same path in each Executor. For example, given a volume defined at \"/vol/A\", executor 1" + + " may write a file to \"/vol/A/exec1.out.txt\", then executor 2 may read from that file. " + + "(Essentially, this translates to a `docker run -v` flag where the container path is the same " + + "for each executor).") + @JsonProperty("volumes") + @Valid + private List volumes = null; + + @ApiModelProperty(value = "A key-value map of arbitrary tags.") + @JsonProperty("tags") + @Valid + private Map tags = null; + @ApiModelProperty(value = "Task logging information. Normally, this will contain only one entry, but in the" + + " case where a task fails and is retried, an entry will be appended to this list.") + @JsonProperty("logs") + @Valid + private List logs = null; + + @ApiModelProperty(value = "Date + time the task was created, in RFC 3339 format. This is set by the system, " + + "not the client.") + @JsonProperty("creation_time") + private String creationTime = null; +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java new file mode 100644 index 0000000000..263158b135 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java @@ -0,0 +1,50 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import javax.validation.Valid; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +@ApiModel(description = "TaskLog describes logging information related to a Task.") +@Data +public class TesTaskLog { + @ApiModelProperty(value = "Logs for each executor") + @JsonProperty("logs") + @Valid + private List logs = null; + + @ApiModelProperty(value = "Arbitrary logging metadata included by the implementation.") + @JsonProperty("metadata") + @Valid + private Map metadata = null; + + @ApiModelProperty(value = "When the task started, in RFC 3339 format.") + @JsonProperty("start_time") + private String startTime = null; + + @ApiModelProperty(value = "When the task ended, in RFC 3339 format.") + @JsonProperty("end_time") + private String endTime = null; + + @ApiModelProperty(value = "Information about all output files. Directory outputs are flattened into separate items.") + @JsonProperty("outputs") + @Valid + private List outputs = null; + + @ApiModelProperty(value = "System logs are any logs the system decides are relevant, which are not tied " + + "directly to an Executor process. Content is implementation specific: format, size, etc. " + + "System logs may be collected here to provide convenient access. For example, the system may " + + "include the name of the host where the task is executing, an error message that caused a SYSTEM_ERROR " + + "state (e.g. disk is full), etc. System logs are only included in the FULL task view.") + @JsonProperty("system_logs") + @Valid + private List systemLogs = null; +} + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java new file mode 100644 index 0000000000..2029596802 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -0,0 +1,7 @@ +package com.epam.pipeline.tesadapter.service; + +import org.springframework.stereotype.Service; + +@Service +public class TesTaskService { +} From adad43915119bb9dbf30dc04dddf12f7e8d10739 Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Tue, 20 Aug 2019 18:51:41 +0300 Subject: [PATCH 013/194] created TesTaskService, injected if to TesAdapterController (#10) * created TesTaskService, injected if to TesAdapterController * refactoring TesTaskService(change from class to interface) * add TesTaskImpl --- .../epam/pipeline/tesadapter/TesAdapterApplication.java | 1 + .../tesadapter/configuration/TesSwaggerConfig.java | 2 -- .../tesadapter/controller/TesAdapterController.java | 7 ++++++- .../epam/pipeline/tesadapter/service/TesTaskService.java | 5 +---- .../pipeline/tesadapter/service/TesTaskServiceImpl.java | 7 +++++++ 5 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java index 9a04a3c380..bc9b5545ff 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java @@ -7,6 +7,7 @@ import org.springframework.context.annotation.Import; @SpringBootApplication +@ComponentScan("com.epam.pipeline.tesadapter.service") @Import({TesSwaggerConfig.class}) @ComponentScan({"com.epam.pipeline.tesadapter.controller", "com.epam.pipeline.tesadapter.service"}) public class TesAdapterApplication { diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java index 6c2887e3cb..1578feb819 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java @@ -10,7 +10,6 @@ import springfox.documentation.spring.web.plugins.Docket; import springfox.documentation.swagger2.annotations.EnableSwagger2; - @Configuration @EnableSwagger2 public class TesSwaggerConfig { @@ -25,7 +24,6 @@ public Docket api() { .build(); } - private ApiInfo getApiInfo() { return new ApiInfoBuilder() .title("TES adapter API Documentation") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index ce42a18220..b7928577cf 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -9,8 +9,13 @@ @RestController public class TesAdapterController { - @Autowired + private TesTaskService tesTaskService; + + @Autowired + public TesAdapterController(TesTaskService tesTaskService) { + this.tesTaskService = tesTaskService; + } @GetMapping("/v1/tasks/service-info") public ResponseEntity serviceInfo() { diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index 2029596802..5c7e1f9efa 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -1,7 +1,4 @@ package com.epam.pipeline.tesadapter.service; -import org.springframework.stereotype.Service; - -@Service -public class TesTaskService { +public interface TesTaskService { } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java new file mode 100644 index 0000000000..b12e12a79c --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -0,0 +1,7 @@ +package com.epam.pipeline.tesadapter.service; + +import org.springframework.stereotype.Service; + +@Service +public class TesTaskServiceImpl implements TesTaskService { +} From fa124e2ddafe669bbe13687b09503e4b51913802 Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Tue, 20 Aug 2019 19:18:28 +0300 Subject: [PATCH 014/194] add MappingJackson2HttpMessageConverter (#6) * add MappingJackson2HttpMessageConverter * refactored MappingJackson2HttpMessageConverter * refactored MappingJackson2HttpMessageConverter(ijected ComponentScan) * refactoring TesSwaggerCongig. Delete unnecessary code and two unnecessary import * refactoring CustomConverter * refactoring code into good Code style --- tes-adapter/build.gradle | 2 ++ .../tesadapter/TesAdapterApplication.java | 6 +++--- .../configuration/RestConfiguration.java | 20 +++++++++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index b89368eb5d..a0fd2b2fd8 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -36,4 +36,6 @@ dependencies { annotationProcessor 'org.projectlombok:lombok:1.18.8' testImplementation('org.junit.jupiter:junit-jupiter-api:5.4.2') testRuntime('org.junit.jupiter:junit-jupiter-engine:5.4.2') + compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.10.0.pr1' + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.0.pr1' } \ No newline at end of file diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java index bc9b5545ff..a0cc62c398 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java @@ -1,5 +1,6 @@ package com.epam.pipeline.tesadapter; +import com.epam.pipeline.tesadapter.configuration.RestConfiguration; import com.epam.pipeline.tesadapter.configuration.TesSwaggerConfig; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -7,9 +8,8 @@ import org.springframework.context.annotation.Import; @SpringBootApplication -@ComponentScan("com.epam.pipeline.tesadapter.service") -@Import({TesSwaggerConfig.class}) -@ComponentScan({"com.epam.pipeline.tesadapter.controller", "com.epam.pipeline.tesadapter.service"}) +@ComponentScan("com.epam.pipeline.tesadapter.controller") +@Import({TesSwaggerConfig.class, RestConfiguration.class}) public class TesAdapterApplication { public static void main(String[] args) { SpringApplication.run(TesAdapterApplication.class, args); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java new file mode 100644 index 0000000000..fa54d6c4b2 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java @@ -0,0 +1,20 @@ +package com.epam.pipeline.tesadapter.configuration; + +import org.springframework.boot.autoconfigure.http.HttpMessageConverters; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +import java.util.Collections; + +@Configuration +public class RestConfiguration implements WebMvcConfigurer { + + @Bean + public HttpMessageConverters customConverters() { + HttpMessageConverter addition = new MappingJackson2HttpMessageConverter(); + return new HttpMessageConverters(false, Collections.singletonList(addition)); + } +} From e9d95b82aa8e6cb0b8456aa1490d62383bc10632 Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Tue, 20 Aug 2019 16:25:48 +0000 Subject: [PATCH 015/194] Minor configuration clean up, PMD and Checkstyle fixes --- tes-adapter/src/main/application.properties | 0 .../tesadapter/TesAdapterApplication.java | 10 ++--- .../controller/TesAdapterController.java | 1 + .../entity/TesCancelTaskResponse.java | 1 - .../entity/TesCreateTaskResponse.java | 3 +- .../tesadapter/entity/TesExecutor.java | 17 +++----- .../tesadapter/entity/TesExecutorLog.java | 13 +++--- .../tesadapter/entity/TesFileType.java | 6 --- .../pipeline/tesadapter/entity/TesInput.java | 13 +++--- .../entity/TesListTasksResponse.java | 6 +-- .../pipeline/tesadapter/entity/TesOutput.java | 11 +++-- .../tesadapter/entity/TesOutputFileLog.java | 7 ++-- .../tesadapter/entity/TesResources.java | 12 +++--- .../tesadapter/entity/TesServiceInfo.java | 12 +++--- .../pipeline/tesadapter/entity/TesState.java | 18 +++++++-- .../pipeline/tesadapter/entity/TesTask.java | 40 +++++++------------ .../tesadapter/entity/TesTaskLog.java | 24 ++++------- .../tesadapter/service/TesTaskService.java | 2 + .../service/TesTaskServiceImpl.java | 5 +++ .../src/main/resources/application.properties | 1 + 20 files changed, 92 insertions(+), 110 deletions(-) delete mode 100644 tes-adapter/src/main/application.properties create mode 100644 tes-adapter/src/main/resources/application.properties diff --git a/tes-adapter/src/main/application.properties b/tes-adapter/src/main/application.properties deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java index a0cc62c398..d9d1926ab6 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java @@ -1,15 +1,15 @@ package com.epam.pipeline.tesadapter; -import com.epam.pipeline.tesadapter.configuration.RestConfiguration; -import com.epam.pipeline.tesadapter.configuration.TesSwaggerConfig; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Import; @SpringBootApplication -@ComponentScan("com.epam.pipeline.tesadapter.controller") -@Import({TesSwaggerConfig.class, RestConfiguration.class}) +@ComponentScan({ + "com.epam.pipeline.tesadapter.configuration", + "com.epam.pipeline.tesadapter.controller", + "com.epam.pipeline.tesadapter.service"}) +@SuppressWarnings("checkstyle:HideUtilityClassConstructor") public class TesAdapterApplication { public static void main(String[] args) { SpringApplication.run(TesAdapterApplication.class, args); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index b7928577cf..0cf8fb103a 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -19,6 +19,7 @@ public TesAdapterController(TesTaskService tesTaskService) { @GetMapping("/v1/tasks/service-info") public ResponseEntity serviceInfo() { + tesTaskService.stub(); return new ResponseEntity<>("OK", HttpStatus.OK); } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java index 47a1f4fd18..e1eb01880c 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCancelTaskResponse.java @@ -3,7 +3,6 @@ import io.swagger.annotations.ApiModel; import lombok.Data; - @ApiModel(description = "CancelTaskResponse describes a response from the CancelTask endpoint.") @Data public class TesCancelTaskResponse { diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java index a558978a78..e3f310b7c0 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesCreateTaskResponse.java @@ -5,12 +5,11 @@ import io.swagger.annotations.ApiModelProperty; import lombok.Data; - @ApiModel(description = "CreateTaskResponse describes a response from the CreateTask endpoint.") @Data public class TesCreateTaskResponse { @ApiModelProperty(value = "Task identifier assigned by the server.") @JsonProperty("id") - private String id = null; + private String id; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java index 4199448ae0..9f90fdbbe9 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java @@ -5,9 +5,6 @@ import io.swagger.annotations.ApiModelProperty; import lombok.Data; -import javax.validation.Valid; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -17,18 +14,17 @@ public class TesExecutor { @ApiModelProperty(value = "Name of the container image, for example: " + "ubuntu quay.io/aptible/ubuntu gcr.io/my-org/my-image etc...") @JsonProperty("image") - private String image = null; + private String image; @ApiModelProperty(value = "A sequence of program arguments to execute, " + "where the first argument is the program to execute (i.e. argv).") @JsonProperty("command") - @Valid - private List command = null; + private List command; @ApiModelProperty(value = "The working directory that the command will be " + "executed in. Defaults to the directory set by the container image.") @JsonProperty("workdir") - private String workdir = null; + private String workdir; @ApiModelProperty(value = "Path inside the container to a file which will " + "be piped to the executor's stdin. Must be an absolute path.") @@ -38,16 +34,15 @@ public class TesExecutor { @ApiModelProperty(value = "Path inside the container to a file where the " + "executor's stdout will be written to. Must be an absolute path.") @JsonProperty("stdout") - private String stdout = null; + private String stdout; @ApiModelProperty(value = "Path inside the container to a file where the " + "executor's stderr will be written to. Must be an absolute path.") @JsonProperty("stderr") - private String stderr = null; + private String stderr; @ApiModelProperty(value = "Enviromental variables to set within the container.") @JsonProperty("env") - @Valid - private Map env = null; + private Map env; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java index cdefa47b03..a5bd614b3b 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java @@ -3,34 +3,35 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.Data; - +@Data @ApiModel(description = "ExecutorLog describes logging information related to an Executor.") public class TesExecutorLog { @ApiModelProperty(value = "Time the executor started, in RFC 3339 format.") @JsonProperty("start_time") - private String startTime = null; + private String startTime; @ApiModelProperty(value = "Time the executor ended, in RFC 3339 format.") @JsonProperty("end_time") - private String endTime = null; + private String endTime; @ApiModelProperty(value = "Stdout content. This is meant for convenience. No guarantees are made about " + "the content. Implementations may chose different approaches: only the head, only the tail, a URL " + "reference only, etc. In order to capture the full stdout users should set Executor.stdout to a " + "container file path, and use Task.outputs to upload that file to permanent storage.") @JsonProperty("stdout") - private String stdout = null; + private String stdout; @ApiModelProperty(value = "Stderr content. This is meant for convenience. No guarantees are made about " + "the content. Implementations may chose different approaches: only the head, only the tail, a URL " + "reference only, etc. In order to capture the full stderr users should set Executor.stderr to a " + "container file path, and use Task.outputs to upload that file to permanent storage.") @JsonProperty("stderr") - private String stderr = null; + private String stderr; @ApiModelProperty(value = "Exit code.") @JsonProperty("exit_code") - private Integer exitCode = null; + private Integer exitCode; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java index bc6bb54a98..291d58c2bf 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesFileType.java @@ -1,17 +1,11 @@ package com.epam.pipeline.tesadapter.entity; -import com.fasterxml.jackson.annotation.JsonValue; - -import com.fasterxml.jackson.annotation.JsonCreator; import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.ToString; /** * Gets or Sets tesFileType */ @AllArgsConstructor -@ToString public enum TesFileType { FILE("FILE"), DIRECTORY("DIRECTORY"); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java index 14759928ef..6d9d5d5c4c 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java @@ -7,37 +7,36 @@ import javax.validation.Valid; - @ApiModel(description = "Input describes Task input files.") @Data public class TesInput { @ApiModelProperty(value = "") @JsonProperty("name") - private String name = null; + private String name; @ApiModelProperty(value = "") @JsonProperty("description") - private String description = null; + private String description; @ApiModelProperty(value = "REQUIRED, unless \"content\" is set. URL in long term storage, for example: " + "s3://my-object-store/file1 gs://my-bucket/file2 file:///path/to/my/file /path/to/my/file etc...") @JsonProperty("url") - private String url = null; + private String url; @ApiModelProperty(value = "Path of the file inside the container. Must be an absolute path.") @JsonProperty("path") - private String path = null; + private String path; @ApiModelProperty(value = "Type of the file, FILE or DIRECTORY") @Valid @JsonProperty("type") - private TesFileType type = null; + private TesFileType type; @ApiModelProperty(value = "File content literal. Implementations should support a minimum of 128 KiB " + "in this field and may define its own maximum. UTF-8 encoded If content is not empty, \"url\" " + "must be ignored.") @JsonProperty("content") - private String content = null; + private String content; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java index 60775dd341..cba3480f38 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java @@ -6,20 +6,18 @@ import lombok.Data; import javax.validation.Valid; -import java.util.ArrayList; import java.util.List; - @ApiModel(description = "ListTasksResponse describes a response from the ListTasks endpoint.") @Data public class TesListTasksResponse { @ApiModelProperty(value = "List of tasks.") @JsonProperty("tasks") @Valid - private List tasks = null; + private List tasks; @ApiModelProperty(value = "Token used to return the next page of results. See TaskListRequest.next_page_token") @JsonProperty("next_page_token") - private String nextPageToken = null; + private String nextPageToken; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java index c96ff556ea..38cccd4138 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java @@ -7,30 +7,29 @@ import javax.validation.Valid; - @ApiModel(description = "Output describes Task output files.") @Data public class TesOutput { @ApiModelProperty(value = "") @JsonProperty("name") - private String name = null; + private String name; @ApiModelProperty(value = "") @JsonProperty("description") - private String description = null; + private String description; @ApiModelProperty(value = "URL in long term storage, for example: s3://my-object-store/file1 " + "gs://my-bucket/file2 file:///path/to/my/file /path/to/my/file etc...") @JsonProperty("url") - private String url = null; + private String url; @ApiModelProperty(value = "Path of the file inside the container. Must be an absolute path.") @JsonProperty("path") - private String path = null; + private String path; @ApiModelProperty(value = "Type of the file, FILE or DIRECTORY") @JsonProperty("type") @Valid - private TesFileType type = null; + private TesFileType type; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java index f9631cde21..9b0b5b5ec6 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutputFileLog.java @@ -5,21 +5,20 @@ import io.swagger.annotations.ApiModelProperty; import lombok.Data; - @ApiModel(description = "OutputFileLog describes a single output file. This describes file details " + "after the task has completed successfully, for logging purposes.") @Data public class TesOutputFileLog { @ApiModelProperty(value = "URL of the file in storage, e.g. s3://bucket/file.txt") @JsonProperty("url") - private String url = null; + private String url; @ApiModelProperty(value = "Path of the file inside the container. Must be an absolute path.") @JsonProperty("path") - private String path = null; + private String path; @ApiModelProperty(value = "Size of the file in bytes.") @JsonProperty("size_bytes") - private String sizeBytes = null; + private String sizeBytes; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java index b07995cdec..6ec7cbb466 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java @@ -6,32 +6,30 @@ import lombok.Data; import javax.validation.Valid; -import java.util.ArrayList; import java.util.List; - @ApiModel(description = "Resources describes the resources requested by a task.") @Data public class TesResources { @ApiModelProperty(value = "Requested number of CPUs") @JsonProperty("cpu_cores") - private Long cpuCores = null; + private Long cpuCores; @ApiModelProperty(value = "Is the task allowed to run on preemptible compute instances (e.g. AWS Spot)?") @JsonProperty("preemptible") - private Boolean preemptible = null; + private Boolean preemptible; @ApiModelProperty(value = "Requested RAM required in gigabytes (GB)") @JsonProperty("ram_gb") - private Double ramGb = null; + private Double ramGb; @ApiModelProperty(value = "Requested disk size in gigabytes (GB)") @JsonProperty("disk_gb") - private Double diskGb = null; + private Double diskGb; @ApiModelProperty(value = "Request that the task be run in these compute zones.") @JsonProperty("zones") @Valid - private List zones = null; + private List zones; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java index 1a6510b9be..29511ccbf5 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesServiceInfo.java @@ -6,27 +6,25 @@ import lombok.Data; import javax.validation.Valid; -import java.util.ArrayList; import java.util.List; - -@ApiModel(description = "ServiceInfo describes information about the service, such as storage details, \r" + - "esource availability, and other documentation.") +@ApiModel(description = "ServiceInfo describes information about the service, such as storage details, " + + "resource availability, and other documentation.") @Data public class TesServiceInfo { @ApiModelProperty(value = "Returns the name of the service, e.g. \"ohsu-compbio-funnel\".") @JsonProperty("name") - private String name = null; + private String name; @ApiModelProperty(value = "Returns a documentation string, e.g. \"Hey, we're OHSU Comp. Bio!\".") @JsonProperty("doc") - private String doc = null; + private String doc; @ApiModelProperty(value = "Lists some, but not necessarily all, storage locations supported by the service. " + " Must be in a valid URL format. e.g. file:///path/to/local/funnel-storage " + "s3://ohsu-compbio-funnel/storage etc.") @JsonProperty("storage") @Valid - private List storage = null; + private List storage; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java index 697eccca14..6cb0fe538b 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesState.java @@ -1,12 +1,24 @@ package com.epam.pipeline.tesadapter.entity; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonValue; import lombok.AllArgsConstructor; import lombok.ToString; /** - * Task states. - UNKNOWN: The state of the task is unknown. This provides a safe default for messages where this field is missing, for example, so that a missing field does not accidentally imply that the state is QUEUED. - QUEUED: The task is queued. - INITIALIZING: The task has been assigned to a worker and is currently preparing to run. For example, the worker may be turning on, downloading input files, etc. - RUNNING: The task is running. Input files are downloaded and the first Executor has been started. - PAUSED: The task is paused. An implementation may have the ability to pause a task, but this is not required. - COMPLETE: The task has completed running. Executors have exited without error and output files have been successfully uploaded. - EXECUTOR_ERROR: The task encountered an error in one of the Executor processes. Generally, this means that an Executor exited with a non-zero exit code. - SYSTEM_ERROR: The task was stopped due to a system error, but not from an Executor, for example an upload failed due to network issues, the worker's ran out of disk space, etc. - CANCELED: The task was canceled by the user. + * Task states. + * - UNKNOWN: The state of the task is unknown. + * This provides a safe default for messages where this field is missing, + * for example, so that a missing field does not accidentally imply that the state is QUEUED. + * - QUEUED: The task is queued. - INITIALIZING: The task has been assigned to a worker and is + * currently preparing to run. For example, the worker may be turning on, downloading input files, etc. + * - RUNNING: The task is running. Input files are downloaded and the first Executor has been started. + * - PAUSED: The task is paused. An implementation may have the ability to pause a task, but this is not required. + * - COMPLETE: The task has completed running. Executors have exited without error and output + * files have been successfully uploaded. + * - EXECUTOR_ERROR: The task encountered an error in one of the Executor processes. + * Generally, this means that an Executor exited with a non-zero exit code. + * - SYSTEM_ERROR: The task was stopped due to a system error, but not from an Executor, + * for example an upload failed due to network issues, the worker's ran out of disk space, etc. + * - CANCELED: The task was canceled by the user. */ @AllArgsConstructor @ToString diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java index 8a8c016f7b..2d5802be62 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java @@ -5,52 +5,44 @@ import io.swagger.annotations.ApiModelProperty; import lombok.Data; -import javax.validation.Valid; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; - @ApiModel(description = "Task describes an instance of a task.") @Data public class TesTask { @ApiModelProperty(value = "Task identifier assigned by the server.") @JsonProperty("id") - private String id = null; + private String id; @ApiModelProperty(value = "") - @Valid @JsonProperty("state") - private TesState state = null; + private TesState state; @ApiModelProperty(value = "") @JsonProperty("name") - private String name = null; + private String name; @ApiModelProperty(value = "") @JsonProperty("description") - private String description = null; + private String description; @ApiModelProperty(value = "Input files. Inputs will be downloaded and mounted into the executor container.") @JsonProperty("inputs") - @Valid - private List inputs = null; + private List inputs; - @ApiModelProperty(value = "Output files. Outputs will be uploaded from the executor container to long-term storage.") + @ApiModelProperty(value = "Output files. Outputs will be uploaded from the executor container " + + "to long-term storage.") @JsonProperty("outputs") - @Valid - private List outputs = null; + private List outputs; @ApiModelProperty(value = "Request that the task be run with these resources.") @JsonProperty("resources") - @Valid - private TesResources resources = null; + private TesResources resources; @ApiModelProperty(value = "A list of executors to be run, sequentially. Execution stops on the first error.") @JsonProperty("executors") - @Valid - private List executors = null; + private List executors; @ApiModelProperty(value = "Volumes are directories which may be used to share data between Executors. " + "Volumes are initialized as empty directories by the system when the task starts and are mounted " + @@ -59,22 +51,20 @@ public class TesTask { "(Essentially, this translates to a `docker run -v` flag where the container path is the same " + "for each executor).") @JsonProperty("volumes") - @Valid - private List volumes = null; + private List volumes; @ApiModelProperty(value = "A key-value map of arbitrary tags.") @JsonProperty("tags") - @Valid - private Map tags = null; + private Map tags; + @ApiModelProperty(value = "Task logging information. Normally, this will contain only one entry, but in the" + " case where a task fails and is retried, an entry will be appended to this list.") @JsonProperty("logs") - @Valid - private List logs = null; + private List logs; @ApiModelProperty(value = "Date + time the task was created, in RFC 3339 format. This is set by the system, " + "not the client.") @JsonProperty("creation_time") - private String creationTime = null; + private String creationTime; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java index 263158b135..b7dd72e7f3 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java @@ -4,39 +4,32 @@ import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; - -import javax.validation.Valid; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; - @ApiModel(description = "TaskLog describes logging information related to a Task.") @Data public class TesTaskLog { @ApiModelProperty(value = "Logs for each executor") @JsonProperty("logs") - @Valid - private List logs = null; + private List logs; @ApiModelProperty(value = "Arbitrary logging metadata included by the implementation.") @JsonProperty("metadata") - @Valid - private Map metadata = null; + private Map metadata; @ApiModelProperty(value = "When the task started, in RFC 3339 format.") @JsonProperty("start_time") - private String startTime = null; + private String startTime; @ApiModelProperty(value = "When the task ended, in RFC 3339 format.") @JsonProperty("end_time") - private String endTime = null; + private String endTime; - @ApiModelProperty(value = "Information about all output files. Directory outputs are flattened into separate items.") + @ApiModelProperty(value = "Information about all output files. " + + "Directory outputs are flattened into separate items.") @JsonProperty("outputs") - @Valid - private List outputs = null; + private List outputs; @ApiModelProperty(value = "System logs are any logs the system decides are relevant, which are not tied " + "directly to an Executor process. Content is implementation specific: format, size, etc. " + @@ -44,7 +37,6 @@ public class TesTaskLog { "include the name of the host where the task is executing, an error message that caused a SYSTEM_ERROR " + "state (e.g. disk is full), etc. System logs are only included in the FULL task view.") @JsonProperty("system_logs") - @Valid - private List systemLogs = null; + private List systemLogs; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index 5c7e1f9efa..a2f9fec66c 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -1,4 +1,6 @@ package com.epam.pipeline.tesadapter.service; public interface TesTaskService { + + void stub(); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index b12e12a79c..04977400ec 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -4,4 +4,9 @@ @Service public class TesTaskServiceImpl implements TesTaskService { + + @Override + public void stub() { + //stubbed method + } } diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties new file mode 100644 index 0000000000..a3ac65cee5 --- /dev/null +++ b/tes-adapter/src/main/resources/application.properties @@ -0,0 +1 @@ +server.port=8080 \ No newline at end of file From 4056ace7d023c5ec9758a354f39b0fa1922a94a2 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 21 Aug 2019 15:40:05 +0300 Subject: [PATCH 016/194] update TesAdapterController. changed import wildcard "*" to single-class imports. Changed settings to not use wildcard import. --- .../tesadapter/controller/TesAdapterController.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index 93c342101b..f4b57abe65 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -7,7 +7,12 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestController; @RestController public class TesAdapterController { From 29cda580a932ebb621ad4e12bcb03ce9ab5dd4aa Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 21 Aug 2019 15:43:35 +0300 Subject: [PATCH 017/194] update TesAdapterController. removed optional value for @PathVariable --- .../pipeline/tesadapter/controller/TesAdapterController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index f4b57abe65..8b56c7920b 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -39,14 +39,14 @@ ResponseEntity submitTesTask(@RequestBody TesTask body) { @GetMapping("/v1/tasks/{id}") @ResponseBody - ResponseEntity getTesTask(@PathVariable("id") String id) { + ResponseEntity getTesTask(@PathVariable String id) { tesTaskService.stub(); return new ResponseEntity(new TesTask(), HttpStatus.NOT_IMPLEMENTED); } @PostMapping("/v1/tasks/{id}:cancel") @ResponseBody - ResponseEntity cancelTesTask(@PathVariable("id") String id) { + ResponseEntity cancelTesTask(@PathVariable String id) { tesTaskService.stub(); return new ResponseEntity(new TesCancelTaskResponse(), HttpStatus.NOT_IMPLEMENTED); } From 4fa52f5c667f0b8f546ba410332f6ab2ab6706ef Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 21 Aug 2019 15:56:17 +0300 Subject: [PATCH 018/194] update TesAdapterController. changed annotation @PathVariable to @RequestParam. Add appropriate modifications to the last one. --- .../tesadapter/controller/TesAdapterController.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index 8b56c7920b..346587d581 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -8,9 +8,9 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; @@ -39,14 +39,15 @@ ResponseEntity submitTesTask(@RequestBody TesTask body) { @GetMapping("/v1/tasks/{id}") @ResponseBody - ResponseEntity getTesTask(@PathVariable String id) { + ResponseEntity getTesTask(@RequestParam String id, @RequestParam(required = false, + defaultValue = "MINIMAL") String view) { tesTaskService.stub(); return new ResponseEntity(new TesTask(), HttpStatus.NOT_IMPLEMENTED); } @PostMapping("/v1/tasks/{id}:cancel") @ResponseBody - ResponseEntity cancelTesTask(@PathVariable String id) { + ResponseEntity cancelTesTask(@RequestParam String id) { tesTaskService.stub(); return new ResponseEntity(new TesCancelTaskResponse(), HttpStatus.NOT_IMPLEMENTED); } From 1c48e7a70a50814b7115abf1fe1d54f8bc2c553b Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Wed, 21 Aug 2019 18:30:30 +0300 Subject: [PATCH 019/194] add EPMCMBIBPC-2764 Task (#13) * add EPMCMBIBPC-2764 Task * add RequestParam into listTesTasks method * add taskView Enum and refactoring TesAdapterControll * refactoring TesAdapterControll --- .../tesadapter/controller/TesAdapterController.java | 12 ++++++++++++ .../epam/pipeline/tesadapter/entity/TaskView.java | 8 ++++++++ .../pipeline/tesadapter/service/TesTaskService.java | 3 +++ .../tesadapter/service/TesTaskServiceImpl.java | 6 ++++++ 4 files changed, 29 insertions(+) create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskView.java diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index 0cf8fb103a..0f3ce7bf1d 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -1,10 +1,13 @@ package com.epam.pipeline.tesadapter.controller; +import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import com.epam.pipeline.tesadapter.entity.TaskView; import com.epam.pipeline.tesadapter.service.TesTaskService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; @RestController @@ -22,4 +25,13 @@ public ResponseEntity serviceInfo() { tesTaskService.stub(); return new ResponseEntity<>("OK", HttpStatus.OK); } + + @GetMapping("/v1/tasks") + public ResponseEntity listTesTasks(@RequestParam(name = "name_prefix", required = false) String namePrefix, + @RequestParam(name = "page_size", required = false) long pageSize, + @RequestParam(name = "page_token", required = false) String pageToken, + @RequestParam(name = "view", required = false, defaultValue = "MINIMAL") + TaskView view) { + return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.listTesTask()); + } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskView.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskView.java new file mode 100644 index 0000000000..ace90a02cd --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskView.java @@ -0,0 +1,8 @@ +package com.epam.pipeline.tesadapter.entity; + +public enum TaskView { + MINIMAL, + BASIC, + FULL; + +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index a2f9fec66c..8e9006f297 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -1,6 +1,9 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; + public interface TesTaskService { + TesListTasksResponse listTesTask(); void stub(); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 04977400ec..f1981705ca 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -1,9 +1,15 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; import org.springframework.stereotype.Service; @Service public class TesTaskServiceImpl implements TesTaskService { + @Override + public TesListTasksResponse listTesTask() { + TesListTasksResponse tesListTasksResponse = new TesListTasksResponse(); + return new TesListTasksResponse(); + } @Override public void stub() { From 99183b214d68b39c2b3297cc142184c8b1670c47 Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Wed, 21 Aug 2019 18:42:11 +0300 Subject: [PATCH 020/194] PMD & checkstyle fix --- .../tesadapter/controller/TesAdapterController.java | 10 +++++----- .../tesadapter/service/TesTaskServiceImpl.java | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index 0f3ce7bf1d..6799542f47 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -27,11 +27,11 @@ public ResponseEntity serviceInfo() { } @GetMapping("/v1/tasks") - public ResponseEntity listTesTasks(@RequestParam(name = "name_prefix", required = false) String namePrefix, - @RequestParam(name = "page_size", required = false) long pageSize, - @RequestParam(name = "page_token", required = false) String pageToken, - @RequestParam(name = "view", required = false, defaultValue = "MINIMAL") - TaskView view) { + public ResponseEntity listTesTasks( + @RequestParam(name = "name_prefix", required = false) String namePrefix, + @RequestParam(name = "page_size", required = false) long pageSize, + @RequestParam(name = "page_token", required = false) String pageToken, + @RequestParam(name = "view", required = false, defaultValue = "MINIMAL") TaskView view) { return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.listTesTask()); } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index f1981705ca..8a756dbd40 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -7,7 +7,6 @@ public class TesTaskServiceImpl implements TesTaskService { @Override public TesListTasksResponse listTesTask() { - TesListTasksResponse tesListTasksResponse = new TesListTasksResponse(); return new TesListTasksResponse(); } From 25c5e7dbd2f7bea9f50cee6d81e0b17114fa7424 Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Thu, 22 Aug 2019 17:54:53 +0300 Subject: [PATCH 021/194] TES exception handler (#14) * simple test tasks service implemented. injected to TesAdapterController and also in ComponentScan * implemented task entities with swagger-codegen and task_execution.swagger.json * added stubbed methods submit/cancel/get-task. Reformatting entities imports. added TES-API interface. * update TesAdapterController. changed import wildcard "*" to single-class imports. Changed settings to not use wildcard import. * update TesAdapterController. removed optional value for @PathVariable * update TesAdapterController. changed annotation @PathVariable to @RequestParam. Add appropriate modifications to the last one. * Added TesExceptionHandler. injected Slf4j logger, HttpStatus stubbed as INTERNAL_SERVER_ERROR. * add EPMCMBIBPC-2764 Task (#13) * add EPMCMBIBPC-2764 Task * add RequestParam into listTesTasks method * add taskView Enum and refactoring TesAdapterControll * refactoring TesAdapterControll * PMD & checkstyle fix * Merging with current state of tes-support remote branch. Add needed changes to Controller. --- .../controller/TesAdapterController.java | 2 +- .../controller/TesExceptionHandler.java | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index 6799542f47..3f6dd41ca0 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -1,7 +1,7 @@ package com.epam.pipeline.tesadapter.controller; -import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; import com.epam.pipeline.tesadapter.entity.TaskView; +import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; import com.epam.pipeline.tesadapter.service.TesTaskService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java new file mode 100644 index 0000000000..eba9e3a111 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java @@ -0,0 +1,21 @@ +package com.epam.pipeline.tesadapter.controller; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.context.request.WebRequest; + +@Slf4j +@RestControllerAdvice("com.epam.pipeline.tesadapter.controller") +public class TesExceptionHandler { + + @ExceptionHandler(Throwable.class) + public final ResponseEntity handleUncaughtException(final Throwable exception, final WebRequest + request) { + log.error(exception.getMessage() + request.getDescription(true)); + return new ResponseEntity<>(exception.getMessage() + request.getDescription(true), + HttpStatus.INTERNAL_SERVER_ERROR); + } +} From fe8308521ab325388ee7a813a9193e80569389c6 Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Thu, 22 Aug 2019 17:55:21 +0300 Subject: [PATCH 022/194] F logger tes (#12) * added new module "tes-adapter" in root directory "cloud-pipeline" * finished sub-task 2759. Added basic Spring Web-Application in tes-adapter module. Create standalone JAR with "OK" response in endpoint "/v1/tasks/service-info" of URL * impl logger to tes-module. Updated build.gradle and added logger-xml config to resources. * removed Logger for saml, via unnecessary reasons. Changed Logger name com.epam.pipeline to com.epam.pipeline.tesadapter * Delete TesAdaptorsController I pushed twice unnecessary file, so I removed it, because we already have proper Controller --- settings.gradle | 3 +- tes-adapter/build.gradle | 3 + tes-adapter/src/main/application.properties | 0 .../src/main/resources/log4j2-spring.xml | 83 +++++++++++++++++++ 4 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 tes-adapter/src/main/application.properties create mode 100644 tes-adapter/src/main/resources/log4j2-spring.xml diff --git a/settings.gradle b/settings.gradle index 5734c1cd7a..8ad94800b6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -14,5 +14,4 @@ include 'vm-monitor' include 'data-sharing-service' include 'data-sharing-service:api' include 'data-sharing-service:client' -include 'tes-adapter' - +include 'tes-adapter' \ No newline at end of file diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index a0fd2b2fd8..447c82d44b 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -38,4 +38,7 @@ dependencies { testRuntime('org.junit.jupiter:junit-jupiter-engine:5.4.2') compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.10.0.pr1' compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.0.pr1' + compile('org.springframework.boot:spring-boot-starter-log4j2'){ + exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging' + } } \ No newline at end of file diff --git a/tes-adapter/src/main/application.properties b/tes-adapter/src/main/application.properties new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tes-adapter/src/main/resources/log4j2-spring.xml b/tes-adapter/src/main/resources/log4j2-spring.xml new file mode 100644 index 0000000000..9ee48e1150 --- /dev/null +++ b/tes-adapter/src/main/resources/log4j2-spring.xml @@ -0,0 +1,83 @@ + + + + + + logs + + + + + + [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n + > + + + + + + + + + + + + + + + + + + [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n + + + + + + + + + + + + + + + + + + + [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n + + + + + + + + + + + + + + + + + From f2757235cee91bcd3def208471c6cd030be905c2 Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Thu, 22 Aug 2019 17:56:04 +0300 Subject: [PATCH 023/194] Added submit/get/cancel-task methods (#9) * simple test tasks service implemented. injected to TesAdapterController and also in ComponentScan * implemented task entities with swagger-codegen and task_execution.swagger.json * added stubbed methods submit/cancel/get-task. Reformatting entities imports. added TES-API interface. * update TesAdapterController. changed import wildcard "*" to single-class imports. Changed settings to not use wildcard import. * update TesAdapterController. removed optional value for @PathVariable * update TesAdapterController. changed annotation @PathVariable to @RequestParam. Add appropriate modifications to the last one. --- .../controller/TesAdapterController.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index 3f6dd41ca0..b5ec950911 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -1,13 +1,19 @@ package com.epam.pipeline.tesadapter.controller; import com.epam.pipeline.tesadapter.entity.TaskView; +import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; +import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.tesadapter.service.TesTaskService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; @RestController @@ -34,4 +40,26 @@ public ResponseEntity listTesTasks( @RequestParam(name = "view", required = false, defaultValue = "MINIMAL") TaskView view) { return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.listTesTask()); } + + @PostMapping("/v1/tasks") + @ResponseBody + ResponseEntity submitTesTask(@RequestBody TesTask body) { + tesTaskService.stub(); + return new ResponseEntity(new TesCreateTaskResponse(), HttpStatus.NOT_IMPLEMENTED); + } + + @GetMapping("/v1/tasks/{id}") + @ResponseBody + ResponseEntity getTesTask(@RequestParam String id, @RequestParam(required = false, + defaultValue = "MINIMAL") TaskView view) { + tesTaskService.stub(); + return new ResponseEntity(new TesTask(), HttpStatus.NOT_IMPLEMENTED); + } + + @PostMapping("/v1/tasks/{id}:cancel") + @ResponseBody + ResponseEntity cancelTesTask(@RequestParam String id) { + tesTaskService.stub(); + return new ResponseEntity(new TesCancelTaskResponse(), HttpStatus.NOT_IMPLEMENTED); + } } From 533cfe77bcdbdd0d85d44f60b20ba5d348dd9248 Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Thu, 22 Aug 2019 20:46:37 +0300 Subject: [PATCH 024/194] add cloud-pipeline-common module in dependencies (#16) implemented CloudPipelineAPIClient --- tes-adapter/build.gradle | 1 + tes-adapter/src/main/application.properties | 3 +++ .../service/CloudPipelineAPIClient.java | 26 +++++++++++++++++++ .../service/TesTaskServiceImpl.java | 8 ++++++ 4 files changed, 38 insertions(+) create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index 447c82d44b..d8ea6cdc95 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -29,6 +29,7 @@ jar { sourceCompatibility = 1.8 dependencies { + implementation project(":cloud-pipeline-common:model") compile('org.springframework.boot:spring-boot-starter-web') compile('io.springfox:springfox-swagger2:2.9.2') compile('io.springfox:springfox-swagger-ui:2.9.2') diff --git a/tes-adapter/src/main/application.properties b/tes-adapter/src/main/application.properties index e69de29bb2..8cf71a4660 100644 --- a/tes-adapter/src/main/application.properties +++ b/tes-adapter/src/main/application.properties @@ -0,0 +1,3 @@ +#Cloud Pipeline API settings +cloud.pipeline.host= +cloud.pipeline.token= \ No newline at end of file diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java new file mode 100644 index 0000000000..6e264a6175 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -0,0 +1,26 @@ +package com.epam.pipeline.tesadapter.service; + +import com.epam.pipeline.client.pipeline.CloudPipelineAPI; +import com.epam.pipeline.client.pipeline.CloudPipelineApiBuilder; +import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.utils.QueryUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +@Service +public class CloudPipelineAPIClient { + private CloudPipelineAPI cloudPipelineAPI; + + public CloudPipelineAPIClient(@Value("${cloud.pipeline.host}") String cloudPipelineHostUrl, + @Value("${cloud.pipeline.token}") String cloudPipelineToken) { + this.cloudPipelineAPI = + new CloudPipelineApiBuilder(0, 0, cloudPipelineHostUrl, cloudPipelineToken) + .buildClient(); + } + + public PipelineRun loadPipelineRun(final Long pipelineRunId) { + return QueryUtils.execute(cloudPipelineAPI.loadPipelineRun(pipelineRunId)); + } + + +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 8a756dbd40..e8cd437443 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -1,10 +1,18 @@ package com.epam.pipeline.tesadapter.service; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @Service public class TesTaskServiceImpl implements TesTaskService { + private final CloudPipelineAPIClient cloudPipelineAPIClient; + + @Autowired + public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient) { + this.cloudPipelineAPIClient = cloudPipelineAPIClient; + } + @Override public TesListTasksResponse listTesTask() { return new TesListTasksResponse(); From a9122fed8fa9ae49f355482d2e2931b18b4a98e0 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 23 Aug 2019 14:11:36 +0300 Subject: [PATCH 025/194] added stubbed implementation for cancel-id mapping --- .../tesadapter/controller/TesAdapterController.java | 2 +- .../epam/pipeline/tesadapter/service/TesTaskService.java | 3 +++ .../pipeline/tesadapter/service/TesTaskServiceImpl.java | 7 +++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index b5ec950911..b9f09050ce 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -59,7 +59,7 @@ ResponseEntity getTesTask(@RequestParam String id, @RequestParam(requir @PostMapping("/v1/tasks/{id}:cancel") @ResponseBody ResponseEntity cancelTesTask(@RequestParam String id) { - tesTaskService.stub(); + tesTaskService.cancelTesTask(id); return new ResponseEntity(new TesCancelTaskResponse(), HttpStatus.NOT_IMPLEMENTED); } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index 8e9006f297..814ebcd614 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -1,9 +1,12 @@ package com.epam.pipeline.tesadapter.service; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import com.epam.pipeline.tesadapter.entity.TesTask; public interface TesTaskService { TesListTasksResponse listTesTask(); void stub(); + + TesTask cancelTesTask(String id); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 8a756dbd40..162ff65ddf 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -1,6 +1,7 @@ package com.epam.pipeline.tesadapter.service; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import com.epam.pipeline.tesadapter.entity.TesTask; import org.springframework.stereotype.Service; @Service @@ -14,4 +15,10 @@ public TesListTasksResponse listTesTask() { public void stub() { //stubbed method } + + @Override + public TesTask cancelTesTask(String id) { + //stubbed + return null; + } } From 68956b2b2c5aa0d455da674fdd5ea903ab9518c7 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 23 Aug 2019 15:55:08 +0300 Subject: [PATCH 026/194] finished mapping for cancel-Id method. --- .../tesadapter/service/TesTaskService.java | 4 ++-- .../service/TesTaskServiceImpl.java | 21 +++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index 814ebcd614..81e69b9b86 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -1,12 +1,12 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; -import com.epam.pipeline.tesadapter.entity.TesTask; public interface TesTaskService { TesListTasksResponse listTesTask(); void stub(); - TesTask cancelTesTask(String id); + TesCancelTaskResponse cancelTesTask(String id); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 0b5bb5abaf..e0548a5d48 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -1,12 +1,15 @@ package com.epam.pipeline.tesadapter.service; import com.epam.pipeline.entity.pipeline.TaskStatus; +import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; import com.epam.pipeline.vo.RunStatusVO; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import com.epam.pipeline.tesadapter.entity.TesTask; import org.springframework.stereotype.Service; +import org.springframework.util.Assert; +@Slf4j @Service public class TesTaskServiceImpl implements TesTaskService { private final CloudPipelineAPIClient cloudPipelineAPIClient; @@ -27,11 +30,21 @@ public void stub() { } @Override - public TesTask cancelTesTask(String id) { + public TesCancelTaskResponse cancelTesTask(String id) { RunStatusVO updateStatus = new RunStatusVO(); updateStatus.setStatus(TaskStatus.STOPPED); - cloudPipelineAPIClient.updateRunStatus(Long.parseLong(id), updateStatus); + cloudPipelineAPIClient.updateRunStatus(checkRunId(id), updateStatus); + return new TesCancelTaskResponse(); + } - return null; + private Long checkRunId(String id) { + Long longRunId = null; + Assert.hasText(id, "INVALID RUN ID"); + try { + longRunId = Long.parseLong(id); + } catch (NumberFormatException e) { + log.error("INVALID RUN ID"); + } + return longRunId; } } From d2df3b334ec4eff96b2c36c617b66a846efdd735 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 23 Aug 2019 15:58:44 +0300 Subject: [PATCH 027/194] finished mapping for cancel-Id method. Changed mathod-name "checkRunId" to "parseRunId" --- .../epam/pipeline/tesadapter/service/TesTaskServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index e0548a5d48..44495a3357 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -33,11 +33,11 @@ public void stub() { public TesCancelTaskResponse cancelTesTask(String id) { RunStatusVO updateStatus = new RunStatusVO(); updateStatus.setStatus(TaskStatus.STOPPED); - cloudPipelineAPIClient.updateRunStatus(checkRunId(id), updateStatus); + cloudPipelineAPIClient.updateRunStatus(parseRunId(id), updateStatus); return new TesCancelTaskResponse(); } - private Long checkRunId(String id) { + private Long parseRunId(String id) { Long longRunId = null; Assert.hasText(id, "INVALID RUN ID"); try { From 5b1ff6ce472ff42b18014460c1c2b54134fc2dc6 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 23 Aug 2019 18:43:27 +0300 Subject: [PATCH 028/194] updated test implementation. Not finished --- tes-adapter/build.gradle | 14 ++++++-- .../app/TesAdapterControllerTest.java | 35 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index 7011bd1c66..1d21843cf3 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -31,16 +31,26 @@ sourceCompatibility = 1.8 dependencies { implementation project(":cloud-pipeline-common:model") compile('org.springframework.boot:spring-boot-starter-web') + + //Swagger compile('io.springfox:springfox-swagger2:2.9.2') compile('io.springfox:springfox-swagger-ui:2.9.2') + + //Lombok compileOnly 'org.projectlombok:lombok:1.18.8' annotationProcessor 'org.projectlombok:lombok:1.18.8' - testImplementation('org.junit.jupiter:junit-jupiter-api:5.4.2') - testRuntime('org.junit.jupiter:junit-jupiter-engine:5.4.2') + + //Jackson compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.10.0.pr1' compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.0.pr1' + + //Logger compile('org.springframework.boot:spring-boot-starter-log4j2'){ exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging' } + //Tests + testImplementation('org.junit.jupiter:junit-jupiter-api:5.4.2') + testRuntime('org.junit.jupiter:junit-jupiter-engine:5.4.2') + testCompile("org.springframework.boot:spring-boot-starter-test") } \ No newline at end of file diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java new file mode 100644 index 0000000000..30d6370fe5 --- /dev/null +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -0,0 +1,35 @@ +package com.epam.pipeline.tesadapter.app; + + +import com.epam.pipeline.tesadapter.controller.TesAdapterController; +import com.epam.pipeline.tesadapter.service.TesTaskServiceImpl; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.test.web.servlet.MockMvc; + +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; + + +@ExtendWith(SpringExtension.class) +@WebMvcTest(TesAdapterController.class) +public class TesAdapterControllerTest { + @Autowired + private MockMvc mockMvc; + + @MockBean + private TesTaskServiceImpl tesTaskService; + + + @Test + public void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { + when(tesTaskService.cancelTesTask("5")).thenReturn(null); + this.mockMvc.perform(get("/v1/tasks/{id}:cancel")).andDo(print()); + } + +} From e5a8dcdcbe43779d7237a1e80bdef654c7e68e3a Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Mon, 26 Aug 2019 17:43:10 +0300 Subject: [PATCH 029/194] Added $API and $API_TOKEN to application.properties. Changed application config in build.gradle and in main TesAdapterApplication class. Reformatting cancel-Id method. --- tes-adapter/build.gradle | 8 ++++---- tes-adapter/src/main/application.properties | 3 --- .../epam/pipeline/tesadapter/TesAdapterApplication.java | 7 +++++++ .../tesadapter/controller/TesAdapterController.java | 7 +++---- tes-adapter/src/main/resources/application.properties | 4 +++- 5 files changed, 17 insertions(+), 12 deletions(-) delete mode 100644 tes-adapter/src/main/application.properties diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index 7011bd1c66..d6c3d94ce0 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -29,6 +29,9 @@ jar { sourceCompatibility = 1.8 dependencies { + configurations { + all*.exclude module : "spring-boot-starter-logging" + } implementation project(":cloud-pipeline-common:model") compile('org.springframework.boot:spring-boot-starter-web') compile('io.springfox:springfox-swagger2:2.9.2') @@ -39,8 +42,5 @@ dependencies { testRuntime('org.junit.jupiter:junit-jupiter-engine:5.4.2') compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.10.0.pr1' compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.0.pr1' - compile('org.springframework.boot:spring-boot-starter-log4j2'){ - exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging' - } - + compile('org.springframework.boot:spring-boot-starter-log4j2') } \ No newline at end of file diff --git a/tes-adapter/src/main/application.properties b/tes-adapter/src/main/application.properties deleted file mode 100644 index 8cf71a4660..0000000000 --- a/tes-adapter/src/main/application.properties +++ /dev/null @@ -1,3 +0,0 @@ -#Cloud Pipeline API settings -cloud.pipeline.host= -cloud.pipeline.token= \ No newline at end of file diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java index d9d1926ab6..56d27da9ab 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java @@ -1,7 +1,11 @@ package com.epam.pipeline.tesadapter; import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.annotation.ComponentScan; @SpringBootApplication @@ -10,6 +14,9 @@ "com.epam.pipeline.tesadapter.controller", "com.epam.pipeline.tesadapter.service"}) @SuppressWarnings("checkstyle:HideUtilityClassConstructor") +@EnableAutoConfiguration(exclude = {DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class}) public class TesAdapterApplication { public static void main(String[] args) { SpringApplication.run(TesAdapterApplication.class, args); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index b9f09050ce..4c102bb3b9 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -18,9 +18,9 @@ @RestController public class TesAdapterController { - + private TesTaskService tesTaskService; - + @Autowired public TesAdapterController(TesTaskService tesTaskService) { this.tesTaskService = tesTaskService; @@ -59,7 +59,6 @@ ResponseEntity getTesTask(@RequestParam String id, @RequestParam(requir @PostMapping("/v1/tasks/{id}:cancel") @ResponseBody ResponseEntity cancelTesTask(@RequestParam String id) { - tesTaskService.cancelTesTask(id); - return new ResponseEntity(new TesCancelTaskResponse(), HttpStatus.NOT_IMPLEMENTED); + return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.cancelTesTask(id)); } } diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index a3ac65cee5..1c344ce654 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -1 +1,3 @@ -server.port=8080 \ No newline at end of file +server.port=8080 +cloud.pipeline.host=${API} +cloud.pipeline.token=${API_TOKEN} \ No newline at end of file From 5c6f62fffd8c70fc39eee7c8315dba4e629178bb Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Mon, 26 Aug 2019 18:30:49 +0300 Subject: [PATCH 030/194] add test TesTaskServiceImpl --- tes-adapter/build.gradle | 9 ++++++++ .../service/CloudPipelineAPIClient.java | 2 -- .../service/TesTaskServiceImplTest.java | 22 +++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index d8ea6cdc95..dbe7616940 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -42,4 +42,13 @@ dependencies { compile('org.springframework.boot:spring-boot-starter-log4j2'){ exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging' } + testImplementation('org.junit.jupiter:junit-jupiter-api:5.2.0') + testCompile('org.junit.jupiter:junit-jupiter-params:5.2.0') + testRuntime('org.junit.jupiter:junit-jupiter-engine:5.2.0') + testCompile "org.mockito:mockito-core:2.+" + testCompile('org.mockito:mockito-junit-jupiter:2.18.3') + testCompile('org.springframework.boot:spring-boot-starter-test') + testImplementation('org.springframework.boot:spring-boot-starter-test') { + exclude group: 'junit', module: 'junit' + } } \ No newline at end of file diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index 6e264a6175..a2a82d8e73 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -21,6 +21,4 @@ public CloudPipelineAPIClient(@Value("${cloud.pipeline.host}") String cloudPipel public PipelineRun loadPipelineRun(final Long pipelineRunId) { return QueryUtils.execute(cloudPipelineAPI.loadPipelineRun(pipelineRunId)); } - - } diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java new file mode 100644 index 0000000000..a62eb44b51 --- /dev/null +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java @@ -0,0 +1,22 @@ +package com.epam.pipeline.tesadapter.service; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +class TesTaskServiceImplTest { + private CloudPipelineAPIClient cloudPipelineAPIClient; + private TesTaskServiceImpl tesTaskService; + + @BeforeAll + void setUp() { + cloudPipelineAPIClient = Mockito.mock(CloudPipelineAPIClient.class); + tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient); + } + + @Test + void listTesTask() { + Assertions.assertNotNull(tesTaskService.listTesTask()); + } +} \ No newline at end of file From b17cb79f87bec772cc99145fa89474102fc8b508 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 27 Aug 2019 14:50:40 +0300 Subject: [PATCH 031/194] refactored build.gradle file. added needed dependencies for junit5 and integration tests. --- tes-adapter/build.gradle | 33 ++++++++++++++++--- .../app/TesAdapterControllerTest.java | 4 +++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index d6c3d94ce0..17498e388c 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -19,6 +19,10 @@ repositories { mavenCentral() } +test { + useJUnitPlatform() +} + jar { manifest { attributes 'Main-Class': 'com.epam.pipeline.tesadapter.TesAdapterApplication' @@ -29,18 +33,37 @@ jar { sourceCompatibility = 1.8 dependencies { - configurations { - all*.exclude module : "spring-boot-starter-logging" - } + //Binding with Pipeline API implementation project(":cloud-pipeline-common:model") + + //Spring compile('org.springframework.boot:spring-boot-starter-web') + + //Swagger compile('io.springfox:springfox-swagger2:2.9.2') compile('io.springfox:springfox-swagger-ui:2.9.2') + + //Lombok compileOnly 'org.projectlombok:lombok:1.18.8' annotationProcessor 'org.projectlombok:lombok:1.18.8' - testImplementation('org.junit.jupiter:junit-jupiter-api:5.4.2') - testRuntime('org.junit.jupiter:junit-jupiter-engine:5.4.2') + + //Jackson compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.10.0.pr1' compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.0.pr1' + + //Logger compile('org.springframework.boot:spring-boot-starter-log4j2') + configurations { + all*.exclude module : "spring-boot-starter-logging" + } + + //Tests + testImplementation('org.junit.jupiter:junit-jupiter-api:5.2.0') + testCompile('org.junit.jupiter:junit-jupiter-params:5.2.0') + testRuntime('org.junit.jupiter:junit-jupiter-engine:5.2.0') + testCompile "org.mockito:mockito-core:2.+" + testCompile('org.mockito:mockito-junit-jupiter:2.18.3') + testImplementation('org.springframework.boot:spring-boot-starter-test') { + exclude group: 'junit', module: 'junit' //direct exclusion junit4 + } } \ No newline at end of file diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 30d6370fe5..c8e9bbf41d 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -2,6 +2,7 @@ import com.epam.pipeline.tesadapter.controller.TesAdapterController; +import com.epam.pipeline.tesadapter.service.CloudPipelineAPIClient; import com.epam.pipeline.tesadapter.service.TesTaskServiceImpl; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -25,6 +26,9 @@ public class TesAdapterControllerTest { @MockBean private TesTaskServiceImpl tesTaskService; + @MockBean + private CloudPipelineAPIClient cloudPipelineAPIClient; + @Test public void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { From d2e348341c812fb72e95772e53f1812da9868cea Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 27 Aug 2019 16:57:48 +0300 Subject: [PATCH 032/194] added simple webMvcTest for cancel-Id method. fixed same method in Controller class, changed annotation @RequestParam to @PathVariable --- .../tesadapter/controller/TesAdapterController.java | 5 +++-- .../tesadapter/app/TesAdapterControllerTest.java | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index 4c102bb3b9..cfdafdddf0 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -10,6 +10,7 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; @@ -50,7 +51,7 @@ ResponseEntity submitTesTask(@RequestBody TesTask body) { @GetMapping("/v1/tasks/{id}") @ResponseBody - ResponseEntity getTesTask(@RequestParam String id, @RequestParam(required = false, + ResponseEntity getTesTask(@PathVariable String id, @RequestParam(required = false, defaultValue = "MINIMAL") TaskView view) { tesTaskService.stub(); return new ResponseEntity(new TesTask(), HttpStatus.NOT_IMPLEMENTED); @@ -58,7 +59,7 @@ ResponseEntity getTesTask(@RequestParam String id, @RequestParam(requir @PostMapping("/v1/tasks/{id}:cancel") @ResponseBody - ResponseEntity cancelTesTask(@RequestParam String id) { + ResponseEntity cancelTesTask(@PathVariable String id) { return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.cancelTesTask(id)); } } diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index c8e9bbf41d..31f9c1c440 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -2,6 +2,7 @@ import com.epam.pipeline.tesadapter.controller.TesAdapterController; +import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.service.CloudPipelineAPIClient; import com.epam.pipeline.tesadapter.service.TesTaskServiceImpl; import org.junit.jupiter.api.Test; @@ -13,8 +14,9 @@ import org.springframework.test.web.servlet.MockMvc; import static org.mockito.Mockito.when; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @ExtendWith(SpringExtension.class) @@ -32,8 +34,9 @@ public class TesAdapterControllerTest { @Test public void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { - when(tesTaskService.cancelTesTask("5")).thenReturn(null); - this.mockMvc.perform(get("/v1/tasks/{id}:cancel")).andDo(print()); + when(tesTaskService.cancelTesTask("5")).thenReturn(new TesCancelTaskResponse()); + this.mockMvc.perform(post("/v1/tasks/{id}:cancel", "5")).andDo(print()) + .andExpect(status().isOk()); } } From 2600ead9a6e9ce965ab629f5f63dbafaedd7f0db Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 27 Aug 2019 17:18:58 +0300 Subject: [PATCH 033/194] added webMvcTest for listTesTask method as a part of integration test for TesAdapterController --- .../pipeline/tesadapter/app/TesAdapterControllerTest.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 31f9c1c440..a451e5553c 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -3,6 +3,7 @@ import com.epam.pipeline.tesadapter.controller.TesAdapterController; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; +import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; import com.epam.pipeline.tesadapter.service.CloudPipelineAPIClient; import com.epam.pipeline.tesadapter.service.TesTaskServiceImpl; import org.junit.jupiter.api.Test; @@ -14,6 +15,7 @@ import org.springframework.test.web.servlet.MockMvc; import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -39,4 +41,10 @@ public void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { .andExpect(status().isOk()); } + @Test + public void listTesTaskWhenRequestingReturnTesListTasksResponse() throws Exception { + when(tesTaskService.listTesTask()).thenReturn(new TesListTasksResponse()); + this.mockMvc.perform(get("/v1/tasks?page_size={size}", 55L)).andDo(print()) + .andExpect(status().isOk()); + } } From 74b32c32893a4fc5dd00c2ab53c1a38b0c4ef9ee Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Wed, 28 Aug 2019 14:07:44 +0300 Subject: [PATCH 034/194] f tes mapping cancel (#17) * simple test tasks service implemented. injected to TesAdapterController and also in ComponentScan * implemented task entities with swagger-codegen and task_execution.swagger.json * added stubbed methods submit/cancel/get-task. Reformatting entities imports. added TES-API interface. * update TesAdapterController. changed import wildcard "*" to single-class imports. Changed settings to not use wildcard import. * update TesAdapterController. removed optional value for @PathVariable * update TesAdapterController. changed annotation @PathVariable to @RequestParam. Add appropriate modifications to the last one. * added stubbed implementation for cancel-id mapping * finished mapping for cancel-Id method. * finished mapping for cancel-Id method. Changed mathod-name "checkRunId" to "parseRunId" * Added $API and $API_TOKEN to application.properties. Changed application config in build.gradle and in main TesAdapterApplication class. Reformatting cancel-Id method. * refactored parseRunId() method in TesTaskServiceImpl. Removed "null" as a possible response(swapped to IllegalArgumentException) * removed local variable in parseRunId method in TesTaskServiceImpl class. * Update TesAdapterController.java Change `@RequestParam` to `@PathVariable` * Update TesAdapterController.java --- tes-adapter/build.gradle | 7 +++--- tes-adapter/src/main/application.properties | 3 --- .../tesadapter/TesAdapterApplication.java | 7 ++++++ .../controller/TesAdapterController.java | 12 +++++----- .../service/CloudPipelineAPIClient.java | 5 ++++ .../tesadapter/service/TesTaskService.java | 3 +++ .../service/TesTaskServiceImpl.java | 24 +++++++++++++++++++ .../src/main/resources/application.properties | 4 +++- 8 files changed, 52 insertions(+), 13 deletions(-) delete mode 100644 tes-adapter/src/main/application.properties diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index d8ea6cdc95..d6c3d94ce0 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -29,6 +29,9 @@ jar { sourceCompatibility = 1.8 dependencies { + configurations { + all*.exclude module : "spring-boot-starter-logging" + } implementation project(":cloud-pipeline-common:model") compile('org.springframework.boot:spring-boot-starter-web') compile('io.springfox:springfox-swagger2:2.9.2') @@ -39,7 +42,5 @@ dependencies { testRuntime('org.junit.jupiter:junit-jupiter-engine:5.4.2') compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.10.0.pr1' compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.0.pr1' - compile('org.springframework.boot:spring-boot-starter-log4j2'){ - exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging' - } + compile('org.springframework.boot:spring-boot-starter-log4j2') } \ No newline at end of file diff --git a/tes-adapter/src/main/application.properties b/tes-adapter/src/main/application.properties deleted file mode 100644 index 8cf71a4660..0000000000 --- a/tes-adapter/src/main/application.properties +++ /dev/null @@ -1,3 +0,0 @@ -#Cloud Pipeline API settings -cloud.pipeline.host= -cloud.pipeline.token= \ No newline at end of file diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java index d9d1926ab6..56d27da9ab 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/TesAdapterApplication.java @@ -1,7 +1,11 @@ package com.epam.pipeline.tesadapter; import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; import org.springframework.context.annotation.ComponentScan; @SpringBootApplication @@ -10,6 +14,9 @@ "com.epam.pipeline.tesadapter.controller", "com.epam.pipeline.tesadapter.service"}) @SuppressWarnings("checkstyle:HideUtilityClassConstructor") +@EnableAutoConfiguration(exclude = {DataSourceAutoConfiguration.class, + DataSourceTransactionManagerAutoConfiguration.class, + HibernateJpaAutoConfiguration.class}) public class TesAdapterApplication { public static void main(String[] args) { SpringApplication.run(TesAdapterApplication.class, args); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index b5ec950911..cfdafdddf0 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -10,6 +10,7 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; @@ -18,9 +19,9 @@ @RestController public class TesAdapterController { - + private TesTaskService tesTaskService; - + @Autowired public TesAdapterController(TesTaskService tesTaskService) { this.tesTaskService = tesTaskService; @@ -50,7 +51,7 @@ ResponseEntity submitTesTask(@RequestBody TesTask body) { @GetMapping("/v1/tasks/{id}") @ResponseBody - ResponseEntity getTesTask(@RequestParam String id, @RequestParam(required = false, + ResponseEntity getTesTask(@PathVariable String id, @RequestParam(required = false, defaultValue = "MINIMAL") TaskView view) { tesTaskService.stub(); return new ResponseEntity(new TesTask(), HttpStatus.NOT_IMPLEMENTED); @@ -58,8 +59,7 @@ ResponseEntity getTesTask(@RequestParam String id, @RequestParam(requir @PostMapping("/v1/tasks/{id}:cancel") @ResponseBody - ResponseEntity cancelTesTask(@RequestParam String id) { - tesTaskService.stub(); - return new ResponseEntity(new TesCancelTaskResponse(), HttpStatus.NOT_IMPLEMENTED); + ResponseEntity cancelTesTask(@PathVariable String id) { + return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.cancelTesTask(id)); } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index 6e264a6175..775836d2e5 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -4,6 +4,7 @@ import com.epam.pipeline.client.pipeline.CloudPipelineApiBuilder; import com.epam.pipeline.entity.pipeline.PipelineRun; import com.epam.pipeline.utils.QueryUtils; +import com.epam.pipeline.vo.RunStatusVO; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -22,5 +23,9 @@ public PipelineRun loadPipelineRun(final Long pipelineRunId) { return QueryUtils.execute(cloudPipelineAPI.loadPipelineRun(pipelineRunId)); } + public PipelineRun updateRunStatus(final Long pipelineRunId, RunStatusVO statusUpdate) { + return QueryUtils.execute(cloudPipelineAPI.updateRunStatus(pipelineRunId, statusUpdate)); + } + } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index 8e9006f297..81e69b9b86 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -1,9 +1,12 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; public interface TesTaskService { TesListTasksResponse listTesTask(); void stub(); + + TesCancelTaskResponse cancelTesTask(String id); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index e8cd437443..e00b760dcf 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -1,9 +1,15 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.entity.pipeline.TaskStatus; +import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import com.epam.pipeline.vo.RunStatusVO; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.util.Assert; +@Slf4j @Service public class TesTaskServiceImpl implements TesTaskService { private final CloudPipelineAPIClient cloudPipelineAPIClient; @@ -22,4 +28,22 @@ public TesListTasksResponse listTesTask() { public void stub() { //stubbed method } + + @Override + public TesCancelTaskResponse cancelTesTask(String id) { + RunStatusVO updateStatus = new RunStatusVO(); + updateStatus.setStatus(TaskStatus.STOPPED); + cloudPipelineAPIClient.updateRunStatus(parseRunId(id), updateStatus); + return new TesCancelTaskResponse(); + } + + private Long parseRunId(String id) { + Assert.hasText(id, "INVALID RUN ID"); + try { + return Long.parseLong(id); + } catch (NumberFormatException e) { + log.error("INVALID RUN ID"); + throw new IllegalArgumentException(e); + } + } } diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index a3ac65cee5..1c344ce654 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -1 +1,3 @@ -server.port=8080 \ No newline at end of file +server.port=8080 +cloud.pipeline.host=${API} +cloud.pipeline.token=${API_TOKEN} \ No newline at end of file From 17e9238b11c1121b2eb8f67c9f0cdaed56ef5399 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 29 Aug 2019 17:19:41 +0300 Subject: [PATCH 035/194] first implementation of submitTesTask() method. fully stubbed --- .../epam/pipeline/tesadapter/service/TesTaskService.java | 5 +++++ .../pipeline/tesadapter/service/TesTaskServiceImpl.java | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index 81e69b9b86..91a86bed95 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -2,10 +2,15 @@ import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import com.epam.pipeline.tesadapter.entity.TesTask; +import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; public interface TesTaskService { + TesCreateTaskResponse submitTesTask(TesTask body); + TesListTasksResponse listTesTask(); + void stub(); TesCancelTaskResponse cancelTesTask(String id); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 44495a3357..43f591c826 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -3,6 +3,8 @@ import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import com.epam.pipeline.tesadapter.entity.TesTask; +import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; import com.epam.pipeline.vo.RunStatusVO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -19,6 +21,12 @@ public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient) { this.cloudPipelineAPIClient = cloudPipelineAPIClient; } + @Override + public TesCreateTaskResponse submitTesTask(TesTask body) { + + return null; + } + @Override public TesListTasksResponse listTesTask() { return new TesListTasksResponse(); From e994fc0373df53faddd60a5baa6ac21a9bc87547 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 29 Aug 2019 17:24:08 +0300 Subject: [PATCH 036/194] moved "magic-numbers" to appropriate constants. --- .../tesadapter/app/TesAdapterControllerTest.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index a451e5553c..14a9082f93 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -24,6 +24,9 @@ @ExtendWith(SpringExtension.class) @WebMvcTest(TesAdapterController.class) public class TesAdapterControllerTest { + private static final String stubbedTaskId = "5"; + private static final Long stubbedPageSize = 55L; + @Autowired private MockMvc mockMvc; @@ -33,18 +36,17 @@ public class TesAdapterControllerTest { @MockBean private CloudPipelineAPIClient cloudPipelineAPIClient; - @Test public void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { - when(tesTaskService.cancelTesTask("5")).thenReturn(new TesCancelTaskResponse()); - this.mockMvc.perform(post("/v1/tasks/{id}:cancel", "5")).andDo(print()) + when(tesTaskService.cancelTesTask(stubbedTaskId)).thenReturn(new TesCancelTaskResponse()); + this.mockMvc.perform(post("/v1/tasks/{id}:cancel", stubbedTaskId)).andDo(print()) .andExpect(status().isOk()); } @Test public void listTesTaskWhenRequestingReturnTesListTasksResponse() throws Exception { when(tesTaskService.listTesTask()).thenReturn(new TesListTasksResponse()); - this.mockMvc.perform(get("/v1/tasks?page_size={size}", 55L)).andDo(print()) + this.mockMvc.perform(get("/v1/tasks?page_size={size}", stubbedPageSize)).andDo(print()) .andExpect(status().isOk()); } } From 0c21fb52e43c3b6aeb446c49d3c9440ff88655b5 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 29 Aug 2019 18:38:43 +0300 Subject: [PATCH 037/194] changed constants view to upper-case --- .../tesadapter/app/TesAdapterControllerTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 14a9082f93..6e917278c5 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -24,8 +24,8 @@ @ExtendWith(SpringExtension.class) @WebMvcTest(TesAdapterController.class) public class TesAdapterControllerTest { - private static final String stubbedTaskId = "5"; - private static final Long stubbedPageSize = 55L; + private static final String STUBBED_TASK_ID = "5"; + private static final Long STUBBED_PAGE_SIZE = 55L; @Autowired private MockMvc mockMvc; @@ -38,15 +38,15 @@ public class TesAdapterControllerTest { @Test public void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { - when(tesTaskService.cancelTesTask(stubbedTaskId)).thenReturn(new TesCancelTaskResponse()); - this.mockMvc.perform(post("/v1/tasks/{id}:cancel", stubbedTaskId)).andDo(print()) + when(tesTaskService.cancelTesTask(STUBBED_TASK_ID)).thenReturn(new TesCancelTaskResponse()); + this.mockMvc.perform(post("/v1/tasks/{id}:cancel", STUBBED_TASK_ID)).andDo(print()) .andExpect(status().isOk()); } @Test public void listTesTaskWhenRequestingReturnTesListTasksResponse() throws Exception { when(tesTaskService.listTesTask()).thenReturn(new TesListTasksResponse()); - this.mockMvc.perform(get("/v1/tasks?page_size={size}", stubbedPageSize)).andDo(print()) + this.mockMvc.perform(get("/v1/tasks?page_size={size}", STUBBED_PAGE_SIZE)).andDo(print()) .andExpect(status().isOk()); } } From 46fa5de1d151f1f6d785d5cf774b1b453e0568c2 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Thu, 29 Aug 2019 19:04:08 +0300 Subject: [PATCH 038/194] add getServiceInfo method --- .../controller/TesAdapterController.java | 6 ++--- .../service/CloudPipelineAPIClient.java | 7 ++++++ .../tesadapter/service/TesTaskService.java | 3 +++ .../service/TesTaskServiceImpl.java | 25 +++++++++++++++++++ 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index cfdafdddf0..a18b36e606 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -4,6 +4,7 @@ import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import com.epam.pipeline.tesadapter.entity.TesServiceInfo; import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.tesadapter.service.TesTaskService; import org.springframework.beans.factory.annotation.Autowired; @@ -28,9 +29,8 @@ public TesAdapterController(TesTaskService tesTaskService) { } @GetMapping("/v1/tasks/service-info") - public ResponseEntity serviceInfo() { - tesTaskService.stub(); - return new ResponseEntity<>("OK", HttpStatus.OK); + public ResponseEntity serviceInfo() { + return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.getServiceInfo()); } @GetMapping("/v1/tasks") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index 775836d2e5..0c77f4ccb2 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -2,12 +2,15 @@ import com.epam.pipeline.client.pipeline.CloudPipelineAPI; import com.epam.pipeline.client.pipeline.CloudPipelineApiBuilder; +import com.epam.pipeline.entity.datastorage.AbstractDataStorage; import com.epam.pipeline.entity.pipeline.PipelineRun; import com.epam.pipeline.utils.QueryUtils; import com.epam.pipeline.vo.RunStatusVO; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import java.util.List; + @Service public class CloudPipelineAPIClient { private CloudPipelineAPI cloudPipelineAPI; @@ -27,5 +30,9 @@ public PipelineRun updateRunStatus(final Long pipelineRunId, RunStatusVO statusU return QueryUtils.execute(cloudPipelineAPI.updateRunStatus(pipelineRunId, statusUpdate)); } + public List loadAllDataStorages(){ + return QueryUtils.execute(cloudPipelineAPI.loadAllDataStorages()); + } + } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index 81e69b9b86..131701fae3 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -2,6 +2,7 @@ import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import com.epam.pipeline.tesadapter.entity.TesServiceInfo; public interface TesTaskService { @@ -9,4 +10,6 @@ public interface TesTaskService { void stub(); TesCancelTaskResponse cancelTesTask(String id); + + TesServiceInfo getServiceInfo(); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index e00b760dcf..3a89627ac6 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -1,14 +1,19 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.entity.datastorage.AbstractDataStorage; import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import com.epam.pipeline.tesadapter.entity.TesServiceInfo; import com.epam.pipeline.vo.RunStatusVO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.Assert; +import java.util.ArrayList; +import java.util.List; + @Slf4j @Service public class TesTaskServiceImpl implements TesTaskService { @@ -46,4 +51,24 @@ private Long parseRunId(String id) { throw new IllegalArgumentException(e); } } + + @Override + public TesServiceInfo getServiceInfo() { + String nameOfService = "CloudPipeline"; + String doc = "https://epam.github.io/cloud-pipeline/"; + TesServiceInfo tesServiceInfo = new TesServiceInfo(); + tesServiceInfo.setName(nameOfService); + tesServiceInfo.setDoc(doc); + tesServiceInfo.setStorage(getDataStorage()); + return tesServiceInfo; + } + + private List getDataStorage(){ + List listPathDataStorage = new ArrayList<>(); + List storageList = cloudPipelineAPIClient.loadAllDataStorages(); + for (AbstractDataStorage i: storageList){ + listPathDataStorage.add(i.getPath()); + } + return listPathDataStorage; + } } From 594446e7042b818796c07c76cd89868307ae1476 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 29 Aug 2019 23:15:45 +0300 Subject: [PATCH 039/194] added ConvertTesTaskToPipelineStart class(need corrections). added appropriate implementations to Controller and Service classes due method submitTesTask. added stubbed properties --- .../client/pipeline/CloudPipelineAPI.java | 4 + .../controller/TesAdapterController.java | 3 +- .../entity/ConvertTesTaskToPipelineStart.java | 87 +++++++++++++++++++ .../service/CloudPipelineAPIClient.java | 5 ++ .../service/TesTaskServiceImpl.java | 4 +- .../src/main/resources/application.properties | 4 +- 6 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/ConvertTesTaskToPipelineStart.java diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java index 9cd2ddf267..8e3f36b049 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java @@ -35,6 +35,7 @@ import com.epam.pipeline.entity.pipeline.RunLog; import com.epam.pipeline.entity.pipeline.Tool; import com.epam.pipeline.entity.pipeline.ToolGroup; +import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.entity.region.AbstractCloudRegion; import com.epam.pipeline.entity.region.AwsRegion; import com.epam.pipeline.entity.security.acl.AclClass; @@ -72,6 +73,9 @@ public interface CloudPipelineAPI { String VERSION = "version"; String PATH = "path"; + @POST("/run") + Call> runPipeline(@Body PipelineStart runVo); + @POST("run/{runId}/status") Call> updateRunStatus(@Path(RUN_ID) Long runId, @Body RunStatusVO statusUpdate); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index cfdafdddf0..b48a8155f7 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -45,8 +45,7 @@ public ResponseEntity listTesTasks( @PostMapping("/v1/tasks") @ResponseBody ResponseEntity submitTesTask(@RequestBody TesTask body) { - tesTaskService.stub(); - return new ResponseEntity(new TesCreateTaskResponse(), HttpStatus.NOT_IMPLEMENTED); + return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.submitTesTask(body)); } @GetMapping("/v1/tasks/{id}") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/ConvertTesTaskToPipelineStart.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/ConvertTesTaskToPipelineStart.java new file mode 100644 index 0000000000..72a47c739b --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/ConvertTesTaskToPipelineStart.java @@ -0,0 +1,87 @@ +package com.epam.pipeline.tesadapter.entity; + +import com.epam.pipeline.entity.configuration.ExecutionEnvironment; +import com.epam.pipeline.entity.configuration.PipeConfValueVO; +import com.epam.pipeline.entity.pipeline.run.PipelineStart; +import com.epam.pipeline.entity.pipeline.run.parameter.RunSid; +import org.springframework.beans.factory.annotation.Value; + +import java.util.List; +import java.util.Map; + +public class ConvertTesTaskToPipelineStart { + private Long pipelineId; + private String version; + private Long timeout; + private String instanceType; + private Integer hddSize; + private String dockerImage; + private String cmdTemplate; + private Long useRunId; + private Long parentNodeId; + private String configurationName; //stubbed + private Integer nodeCount; + private String workerCmd; + private Long parentRunId; + private Boolean isSpot; + private List runSids; + private Long cloudRegionId; //stubbed + private boolean force; + private ExecutionEnvironment executionEnvironment; + private String prettyUrl; + private boolean nonPause; + private Map params; + + private PipelineStart pipelineStart; + + public ConvertTesTaskToPipelineStart(@Value("${cloud.pipeline.instanceType}") String instanceType, + @Value("${cloud.pipeline.hddSize}") Integer hddSize, TesTask tesTask) { + this.pipelineId = null; + this.version = null; + this.timeout = null; + this.instanceType = instanceType; + this.hddSize = hddSize; + this.dockerImage = tesTask.getExecutors().get(0).getImage(); + this.cmdTemplate = String.join(" ", tesTask.getExecutors().get(0).getCommand());//List of Executors? + this.useRunId = null; + this.parentNodeId = null; + this.nodeCount = null; + this.workerCmd = null; + this.parentRunId = null; + this.isSpot = tesTask.getResources().getPreemptible(); + this.runSids = null; + this.force = false; + this.executionEnvironment = ExecutionEnvironment.CLOUD_PLATFORM; + this.prettyUrl = null; + this.nonPause = true; + this.params.put("", tesTask.getInputs().get(0).getContent()); + } + + public void getPipelineStartFromTesTask() { + + } + + private PipelineStart setParamsToPipelineStart() { + pipelineStart.setPipelineId(pipelineId); + pipelineStart.setVersion(version); + pipelineStart.setTimeout(timeout); + pipelineStart.setInstanceType(instanceType); + pipelineStart.setHddSize(hddSize); + pipelineStart.setDockerImage(dockerImage); + pipelineStart.setCmdTemplate(cmdTemplate); + pipelineStart.setUseRunId(useRunId); + pipelineStart.setParentNodeId(parentNodeId); + pipelineStart.setNodeCount(nodeCount); + pipelineStart.setWorkerCmd(workerCmd); + pipelineStart.setParentRunId(parentRunId); + pipelineStart.setIsSpot(isSpot); + pipelineStart.setRunSids(runSids); + pipelineStart.setForce(force); + pipelineStart.setExecutionEnvironment(executionEnvironment); + pipelineStart.setPrettyUrl(prettyUrl); + pipelineStart.setNonPause(nonPause); + pipelineStart.setParams(); + } + + +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index 775836d2e5..ab2ec13e53 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -3,6 +3,7 @@ import com.epam.pipeline.client.pipeline.CloudPipelineAPI; import com.epam.pipeline.client.pipeline.CloudPipelineApiBuilder; import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.utils.QueryUtils; import com.epam.pipeline.vo.RunStatusVO; import org.springframework.beans.factory.annotation.Value; @@ -19,6 +20,10 @@ public CloudPipelineAPIClient(@Value("${cloud.pipeline.host}") String cloudPipel .buildClient(); } + public PipelineRun runPipeline(PipelineStart runVo) { + return QueryUtils.execute(cloudPipelineAPI.runPipeline(runVo)); + } + public PipelineRun loadPipelineRun(final Long pipelineRunId) { return QueryUtils.execute(cloudPipelineAPI.loadPipelineRun(pipelineRunId)); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 43f591c826..df6fb02ab9 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -1,6 +1,7 @@ package com.epam.pipeline.tesadapter.service; import com.epam.pipeline.entity.pipeline.TaskStatus; +import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; import com.epam.pipeline.tesadapter.entity.TesTask; @@ -23,7 +24,8 @@ public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient) { @Override public TesCreateTaskResponse submitTesTask(TesTask body) { - + //stubbed + cloudPipelineAPIClient.runPipeline(new PipelineStart()); return null; } diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index 1c344ce654..7d917418cb 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -1,3 +1,5 @@ server.port=8080 cloud.pipeline.host=${API} -cloud.pipeline.token=${API_TOKEN} \ No newline at end of file +cloud.pipeline.token=${API_TOKEN} +cloud.pipeline.instanceType= +cloud.pipeline.hddSize= \ No newline at end of file From 7018cadbb66255a372497d0f61632cb64385a7ce Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 30 Aug 2019 14:03:00 +0300 Subject: [PATCH 040/194] updated mapping settings for submitTesTask method. class TaskMapper. Need to specify application.properties, and check mapping --- ...skToPipelineStart.java => TaskMapper.java} | 28 +++++++++++-------- .../service/TesTaskServiceImpl.java | 13 +++++---- .../src/main/resources/application.properties | 4 +-- 3 files changed, 26 insertions(+), 19 deletions(-) rename tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/{ConvertTesTaskToPipelineStart.java => TaskMapper.java} (77%) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/ConvertTesTaskToPipelineStart.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskMapper.java similarity index 77% rename from tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/ConvertTesTaskToPipelineStart.java rename to tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskMapper.java index 72a47c739b..d32a9227c0 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/ConvertTesTaskToPipelineStart.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskMapper.java @@ -9,7 +9,7 @@ import java.util.List; import java.util.Map; -public class ConvertTesTaskToPipelineStart { +public class TaskMapper { private Long pipelineId; private String version; private Long timeout; @@ -34,34 +34,40 @@ public class ConvertTesTaskToPipelineStart { private PipelineStart pipelineStart; - public ConvertTesTaskToPipelineStart(@Value("${cloud.pipeline.instanceType}") String instanceType, - @Value("${cloud.pipeline.hddSize}") Integer hddSize, TesTask tesTask) { + public TaskMapper(@Value("${cloud.pipeline.instanceType}") String instanceType, + @Value("${cloud.pipeline.hddSize}") Integer hddSize) { this.pipelineId = null; this.version = null; this.timeout = null; this.instanceType = instanceType; this.hddSize = hddSize; - this.dockerImage = tesTask.getExecutors().get(0).getImage(); - this.cmdTemplate = String.join(" ", tesTask.getExecutors().get(0).getCommand());//List of Executors? this.useRunId = null; this.parentNodeId = null; this.nodeCount = null; this.workerCmd = null; this.parentRunId = null; - this.isSpot = tesTask.getResources().getPreemptible(); this.runSids = null; this.force = false; this.executionEnvironment = ExecutionEnvironment.CLOUD_PLATFORM; this.prettyUrl = null; this.nonPause = true; - this.params.put("", tesTask.getInputs().get(0).getContent()); - } - public void getPipelineStartFromTesTask() { + } + public PipelineStart mapToPipelineStart(TesTask tesTask) { + this.dockerImage = tesTask.getExecutors().get(0).getImage(); //List of Executors + this.cmdTemplate = String.join(" ", tesTask.getExecutors().get(0).getCommand());//List of Executors? + this.isSpot = tesTask.getResources().getPreemptible(); + //map params + this.params.put("inputs", new PipeConfValueVO(tesTask.getInputs().get(0).getPath(), "input")); + this.params.put("outputs", new PipeConfValueVO(tesTask.getOutputs().get(0).getPath(), "output")); + this.params.put("envs", new PipeConfValueVO( + String.join(" ", tesTask.getExecutors().get(0).getEnv().values()), "string")); + setParamsToPipelineStart(); + return this.pipelineStart; } - private PipelineStart setParamsToPipelineStart() { + private void setParamsToPipelineStart() { pipelineStart.setPipelineId(pipelineId); pipelineStart.setVersion(version); pipelineStart.setTimeout(timeout); @@ -80,7 +86,7 @@ private PipelineStart setParamsToPipelineStart() { pipelineStart.setExecutionEnvironment(executionEnvironment); pipelineStart.setPrettyUrl(prettyUrl); pipelineStart.setNonPause(nonPause); - pipelineStart.setParams(); + pipelineStart.setParams(params); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index df6fb02ab9..3092c4fd94 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -1,11 +1,11 @@ package com.epam.pipeline.tesadapter.service; import com.epam.pipeline.entity.pipeline.TaskStatus; -import com.epam.pipeline.entity.pipeline.run.PipelineStart; +import com.epam.pipeline.tesadapter.entity.TaskMapper; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; +import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; import com.epam.pipeline.tesadapter.entity.TesTask; -import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; import com.epam.pipeline.vo.RunStatusVO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -16,17 +16,18 @@ @Service public class TesTaskServiceImpl implements TesTaskService { private final CloudPipelineAPIClient cloudPipelineAPIClient; + private final TaskMapper taskMapper; @Autowired - public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient) { + public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient, TaskMapper taskMapper) { this.cloudPipelineAPIClient = cloudPipelineAPIClient; + this.taskMapper = taskMapper; } @Override public TesCreateTaskResponse submitTesTask(TesTask body) { - //stubbed - cloudPipelineAPIClient.runPipeline(new PipelineStart()); - return null; + cloudPipelineAPIClient.runPipeline(taskMapper.mapToPipelineStart(body)); + return new TesCreateTaskResponse(); } @Override diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index 7d917418cb..e3820d83e2 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -1,5 +1,5 @@ server.port=8080 cloud.pipeline.host=${API} cloud.pipeline.token=${API_TOKEN} -cloud.pipeline.instanceType= -cloud.pipeline.hddSize= \ No newline at end of file +cloud.pipeline.instanceType=large +cloud.pipeline.hddSize=15 \ No newline at end of file From fffb81d6fcc6a67cc781ada9280178c0d910d999 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 30 Aug 2019 14:57:27 +0300 Subject: [PATCH 041/194] fixed run trouble, moved TaskMapper @Value annotation to appropriate fields. fixed ServiceImpl autowired constructor --- .../pipeline/tesadapter/entity/TaskMapper.java | 17 ++++++++--------- .../tesadapter/service/TesTaskServiceImpl.java | 4 ++-- .../src/main/resources/application.properties | 4 ++-- .../app/TesAdapterControllerTest.java | 1 + 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskMapper.java index d32a9227c0..b57ef3a7c5 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskMapper.java @@ -13,19 +13,24 @@ public class TaskMapper { private Long pipelineId; private String version; private Long timeout; + + @Value("${cloud.pipeline.instanceType}") private String instanceType; + + @Value("${cloud.pipeline.hddSize}") private Integer hddSize; + private String dockerImage; private String cmdTemplate; private Long useRunId; private Long parentNodeId; - private String configurationName; //stubbed + // private String configurationName; //stubbed private Integer nodeCount; private String workerCmd; private Long parentRunId; private Boolean isSpot; private List runSids; - private Long cloudRegionId; //stubbed + // private Long cloudRegionId; //stubbed private boolean force; private ExecutionEnvironment executionEnvironment; private String prettyUrl; @@ -34,13 +39,10 @@ public class TaskMapper { private PipelineStart pipelineStart; - public TaskMapper(@Value("${cloud.pipeline.instanceType}") String instanceType, - @Value("${cloud.pipeline.hddSize}") Integer hddSize) { + public TaskMapper() { this.pipelineId = null; this.version = null; this.timeout = null; - this.instanceType = instanceType; - this.hddSize = hddSize; this.useRunId = null; this.parentNodeId = null; this.nodeCount = null; @@ -51,7 +53,6 @@ public TaskMapper(@Value("${cloud.pipeline.instanceType}") String instanceType, this.executionEnvironment = ExecutionEnvironment.CLOUD_PLATFORM; this.prettyUrl = null; this.nonPause = true; - } public PipelineStart mapToPipelineStart(TesTask tesTask) { @@ -88,6 +89,4 @@ private void setParamsToPipelineStart() { pipelineStart.setNonPause(nonPause); pipelineStart.setParams(params); } - - } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 09f6d89b4f..cb8c753040 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -19,9 +19,9 @@ public class TesTaskServiceImpl implements TesTaskService { private final TaskMapper taskMapper; @Autowired - public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient, TaskMapper taskMapper) { + public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient) { this.cloudPipelineAPIClient = cloudPipelineAPIClient; - this.taskMapper = taskMapper; + this.taskMapper = new TaskMapper(); } @Override diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index e3820d83e2..fa6a2901ca 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -1,5 +1,5 @@ server.port=8080 cloud.pipeline.host=${API} cloud.pipeline.token=${API_TOKEN} -cloud.pipeline.instanceType=large -cloud.pipeline.hddSize=15 \ No newline at end of file +cloud.pipeline.instanceType=m5.large +cloud.pipeline.hddSize=50 \ No newline at end of file diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 6e917278c5..70a8fd56eb 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -23,6 +23,7 @@ @ExtendWith(SpringExtension.class) @WebMvcTest(TesAdapterController.class) +@SuppressWarnings("unused") public class TesAdapterControllerTest { private static final String STUBBED_TASK_ID = "5"; private static final Long STUBBED_PAGE_SIZE = 55L; From c970209a5f8ed51a24359f739447b5fde3a2066b Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Fri, 30 Aug 2019 15:10:35 +0300 Subject: [PATCH 042/194] simple integration test for WebMvc (#19) * simple test tasks service implemented. injected to TesAdapterController and also in ComponentScan * implemented task entities with swagger-codegen and task_execution.swagger.json * added stubbed methods submit/cancel/get-task. Reformatting entities imports. added TES-API interface. * update TesAdapterController. changed import wildcard "*" to single-class imports. Changed settings to not use wildcard import. * update TesAdapterController. removed optional value for @PathVariable * update TesAdapterController. changed annotation @PathVariable to @RequestParam. Add appropriate modifications to the last one. * added stubbed implementation for cancel-id mapping * finished mapping for cancel-Id method. * finished mapping for cancel-Id method. Changed mathod-name "checkRunId" to "parseRunId" * updated test implementation. Not finished * Added $API and $API_TOKEN to application.properties. Changed application config in build.gradle and in main TesAdapterApplication class. Reformatting cancel-Id method. * refactored build.gradle file. added needed dependencies for junit5 and integration tests. * added simple webMvcTest for cancel-Id method. fixed same method in Controller class, changed annotation @RequestParam to @PathVariable * added webMvcTest for listTesTask method as a part of integration test for TesAdapterController * moved "magic-numbers" to appropriate constants. * changed constants view to upper-case --- tes-adapter/build.gradle | 33 ++++++++++-- .../app/TesAdapterControllerTest.java | 52 +++++++++++++++++++ 2 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index d6c3d94ce0..17498e388c 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -19,6 +19,10 @@ repositories { mavenCentral() } +test { + useJUnitPlatform() +} + jar { manifest { attributes 'Main-Class': 'com.epam.pipeline.tesadapter.TesAdapterApplication' @@ -29,18 +33,37 @@ jar { sourceCompatibility = 1.8 dependencies { - configurations { - all*.exclude module : "spring-boot-starter-logging" - } + //Binding with Pipeline API implementation project(":cloud-pipeline-common:model") + + //Spring compile('org.springframework.boot:spring-boot-starter-web') + + //Swagger compile('io.springfox:springfox-swagger2:2.9.2') compile('io.springfox:springfox-swagger-ui:2.9.2') + + //Lombok compileOnly 'org.projectlombok:lombok:1.18.8' annotationProcessor 'org.projectlombok:lombok:1.18.8' - testImplementation('org.junit.jupiter:junit-jupiter-api:5.4.2') - testRuntime('org.junit.jupiter:junit-jupiter-engine:5.4.2') + + //Jackson compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.10.0.pr1' compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.0.pr1' + + //Logger compile('org.springframework.boot:spring-boot-starter-log4j2') + configurations { + all*.exclude module : "spring-boot-starter-logging" + } + + //Tests + testImplementation('org.junit.jupiter:junit-jupiter-api:5.2.0') + testCompile('org.junit.jupiter:junit-jupiter-params:5.2.0') + testRuntime('org.junit.jupiter:junit-jupiter-engine:5.2.0') + testCompile "org.mockito:mockito-core:2.+" + testCompile('org.mockito:mockito-junit-jupiter:2.18.3') + testImplementation('org.springframework.boot:spring-boot-starter-test') { + exclude group: 'junit', module: 'junit' //direct exclusion junit4 + } } \ No newline at end of file diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java new file mode 100644 index 0000000000..6e917278c5 --- /dev/null +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -0,0 +1,52 @@ +package com.epam.pipeline.tesadapter.app; + + +import com.epam.pipeline.tesadapter.controller.TesAdapterController; +import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; +import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import com.epam.pipeline.tesadapter.service.CloudPipelineAPIClient; +import com.epam.pipeline.tesadapter.service.TesTaskServiceImpl; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.test.web.servlet.MockMvc; + +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + + +@ExtendWith(SpringExtension.class) +@WebMvcTest(TesAdapterController.class) +public class TesAdapterControllerTest { + private static final String STUBBED_TASK_ID = "5"; + private static final Long STUBBED_PAGE_SIZE = 55L; + + @Autowired + private MockMvc mockMvc; + + @MockBean + private TesTaskServiceImpl tesTaskService; + + @MockBean + private CloudPipelineAPIClient cloudPipelineAPIClient; + + @Test + public void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { + when(tesTaskService.cancelTesTask(STUBBED_TASK_ID)).thenReturn(new TesCancelTaskResponse()); + this.mockMvc.perform(post("/v1/tasks/{id}:cancel", STUBBED_TASK_ID)).andDo(print()) + .andExpect(status().isOk()); + } + + @Test + public void listTesTaskWhenRequestingReturnTesListTasksResponse() throws Exception { + when(tesTaskService.listTesTask()).thenReturn(new TesListTasksResponse()); + this.mockMvc.perform(get("/v1/tasks?page_size={size}", STUBBED_PAGE_SIZE)).andDo(print()) + .andExpect(status().isOk()); + } +} From 9b578ba7c279ab08999016468996335c48a6ae90 Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Fri, 30 Aug 2019 16:37:04 +0300 Subject: [PATCH 043/194] PMD fix for tests --- .../epam/pipeline/tesadapter/app/TesAdapterControllerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 6e917278c5..0eaad644bf 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -34,7 +34,7 @@ public class TesAdapterControllerTest { private TesTaskServiceImpl tesTaskService; @MockBean - private CloudPipelineAPIClient cloudPipelineAPIClient; + CloudPipelineAPIClient cloudPipelineAPIClient; @Test public void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { From f892fee3cd23c249c0a1caceead7f40e39d0ce01 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 30 Aug 2019 18:45:59 +0300 Subject: [PATCH 044/194] finished fully operable method submitTesTask(), which returns TesCreateTaskResponse with appropriate id. added needed modifications to "cloud-pipeline-common" package in CloudPipelineAPI. Need MVC tests for this method. --- .../client/pipeline/CloudPipelineAPI.java | 2 +- .../tesadapter/entity/TaskMapper.java | 92 ------------------- .../tesadapter/service/TaskMapper.java | 67 ++++++++++++++ .../service/TesTaskServiceImpl.java | 14 ++- .../src/main/resources/application.properties | 2 +- 5 files changed, 78 insertions(+), 99 deletions(-) delete mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskMapper.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java index 8e3f36b049..6850ee47a5 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java @@ -73,7 +73,7 @@ public interface CloudPipelineAPI { String VERSION = "version"; String PATH = "path"; - @POST("/run") + @POST("run") Call> runPipeline(@Body PipelineStart runVo); @POST("run/{runId}/status") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskMapper.java deleted file mode 100644 index b57ef3a7c5..0000000000 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskMapper.java +++ /dev/null @@ -1,92 +0,0 @@ -package com.epam.pipeline.tesadapter.entity; - -import com.epam.pipeline.entity.configuration.ExecutionEnvironment; -import com.epam.pipeline.entity.configuration.PipeConfValueVO; -import com.epam.pipeline.entity.pipeline.run.PipelineStart; -import com.epam.pipeline.entity.pipeline.run.parameter.RunSid; -import org.springframework.beans.factory.annotation.Value; - -import java.util.List; -import java.util.Map; - -public class TaskMapper { - private Long pipelineId; - private String version; - private Long timeout; - - @Value("${cloud.pipeline.instanceType}") - private String instanceType; - - @Value("${cloud.pipeline.hddSize}") - private Integer hddSize; - - private String dockerImage; - private String cmdTemplate; - private Long useRunId; - private Long parentNodeId; - // private String configurationName; //stubbed - private Integer nodeCount; - private String workerCmd; - private Long parentRunId; - private Boolean isSpot; - private List runSids; - // private Long cloudRegionId; //stubbed - private boolean force; - private ExecutionEnvironment executionEnvironment; - private String prettyUrl; - private boolean nonPause; - private Map params; - - private PipelineStart pipelineStart; - - public TaskMapper() { - this.pipelineId = null; - this.version = null; - this.timeout = null; - this.useRunId = null; - this.parentNodeId = null; - this.nodeCount = null; - this.workerCmd = null; - this.parentRunId = null; - this.runSids = null; - this.force = false; - this.executionEnvironment = ExecutionEnvironment.CLOUD_PLATFORM; - this.prettyUrl = null; - this.nonPause = true; - } - - public PipelineStart mapToPipelineStart(TesTask tesTask) { - this.dockerImage = tesTask.getExecutors().get(0).getImage(); //List of Executors - this.cmdTemplate = String.join(" ", tesTask.getExecutors().get(0).getCommand());//List of Executors? - this.isSpot = tesTask.getResources().getPreemptible(); - //map params - this.params.put("inputs", new PipeConfValueVO(tesTask.getInputs().get(0).getPath(), "input")); - this.params.put("outputs", new PipeConfValueVO(tesTask.getOutputs().get(0).getPath(), "output")); - this.params.put("envs", new PipeConfValueVO( - String.join(" ", tesTask.getExecutors().get(0).getEnv().values()), "string")); - setParamsToPipelineStart(); - return this.pipelineStart; - } - - private void setParamsToPipelineStart() { - pipelineStart.setPipelineId(pipelineId); - pipelineStart.setVersion(version); - pipelineStart.setTimeout(timeout); - pipelineStart.setInstanceType(instanceType); - pipelineStart.setHddSize(hddSize); - pipelineStart.setDockerImage(dockerImage); - pipelineStart.setCmdTemplate(cmdTemplate); - pipelineStart.setUseRunId(useRunId); - pipelineStart.setParentNodeId(parentNodeId); - pipelineStart.setNodeCount(nodeCount); - pipelineStart.setWorkerCmd(workerCmd); - pipelineStart.setParentRunId(parentRunId); - pipelineStart.setIsSpot(isSpot); - pipelineStart.setRunSids(runSids); - pipelineStart.setForce(force); - pipelineStart.setExecutionEnvironment(executionEnvironment); - pipelineStart.setPrettyUrl(prettyUrl); - pipelineStart.setNonPause(nonPause); - pipelineStart.setParams(params); - } -} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java new file mode 100644 index 0000000000..2c669262af --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -0,0 +1,67 @@ +package com.epam.pipeline.tesadapter.service; + +import com.epam.pipeline.entity.configuration.ExecutionEnvironment; +import com.epam.pipeline.entity.configuration.PipeConfValueVO; +import com.epam.pipeline.entity.pipeline.run.PipelineStart; +import com.epam.pipeline.tesadapter.entity.TesExecutor; +import com.epam.pipeline.tesadapter.entity.TesTask; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.collections4.MapUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +@Component +public class TaskMapper { + private final String defaultInstanceType; + private final Integer defaultHddSize; + + public TaskMapper(@Value("${cloud.pipeline.instanceType}") String instanceType, + @Value("${cloud.pipeline.hddSize}") Integer hddSize) { + this.defaultInstanceType = instanceType; + this.defaultHddSize = hddSize; + } + + public PipelineStart mapToPipelineStart(TesTask tesTask) { + PipelineStart pipelineStart = new PipelineStart(); + Map params = new HashMap<>(); + TesExecutor tesExecutor = getExecutorFromTesExecutorsList(ListUtils.emptyIfNull(tesTask.getExecutors())); + + pipelineStart.setCmdTemplate(String.join(" ", tesExecutor.getCommand())); + pipelineStart.setDockerImage(tesExecutor.getImage()); + pipelineStart.setExecutionEnvironment(ExecutionEnvironment.CLOUD_PLATFORM); + pipelineStart.setHddSize(defaultHddSize); + pipelineStart.setInstanceType(defaultInstanceType); + pipelineStart.setIsSpot(tesTask.getResources().getPreemptible()); + pipelineStart.setForce(false); + pipelineStart.setNonPause(true); + + ListUtils.emptyIfNull(tesTask.getInputs()) + .forEach(tesInput -> + params.put(tesInput.getName(), new PipeConfValueVO(tesInput.getUrl(), "input"))); + + ListUtils.emptyIfNull(tesTask.getOutputs()).forEach(tesOutput -> + params.put(tesOutput.getName(), new PipeConfValueVO(tesOutput.getUrl(), "output"))); + + MapUtils.emptyIfNull(tesExecutor.getEnv()).forEach((name, value) -> + params.put(name, new PipeConfValueVO(value, "string"))); + pipelineStart.setParams(params); + + return pipelineStart; + } + + private TesExecutor getExecutorFromTesExecutorsList(List tesExecutors) { + if (CollectionUtils.isEmpty(tesExecutors)) { + log.error("LIST OF EXECUTORS EMPTY OR NULL"); + throw new IllegalArgumentException(); + } else { + return tesExecutors.get(0); + } + } +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index cb8c753040..0153aaffdf 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -1,7 +1,7 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.entity.pipeline.PipelineRun; import com.epam.pipeline.entity.pipeline.TaskStatus; -import com.epam.pipeline.tesadapter.entity.TaskMapper; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; @@ -19,15 +19,19 @@ public class TesTaskServiceImpl implements TesTaskService { private final TaskMapper taskMapper; @Autowired - public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient) { + public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient, TaskMapper taskMapper) { this.cloudPipelineAPIClient = cloudPipelineAPIClient; - this.taskMapper = new TaskMapper(); + this.taskMapper = taskMapper; } @Override public TesCreateTaskResponse submitTesTask(TesTask body) { - cloudPipelineAPIClient.runPipeline(taskMapper.mapToPipelineStart(body)); - return new TesCreateTaskResponse(); + TesCreateTaskResponse tesCreateTaskResponse = new TesCreateTaskResponse(); + PipelineRun pipelineRun = cloudPipelineAPIClient.runPipeline(taskMapper.mapToPipelineStart(body)); + Assert.notNull(pipelineRun.getId(), "INVALID PIPELINE ID"); + String runId = pipelineRun.getId().toString(); + tesCreateTaskResponse.setId(runId); + return tesCreateTaskResponse; } @Override diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index fa6a2901ca..6027873e50 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -2,4 +2,4 @@ server.port=8080 cloud.pipeline.host=${API} cloud.pipeline.token=${API_TOKEN} cloud.pipeline.instanceType=m5.large -cloud.pipeline.hddSize=50 \ No newline at end of file +cloud.pipeline.hddSize=20 \ No newline at end of file From 281fec55d29775ce1f57b34b5791cb287cbd8475 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 30 Aug 2019 18:49:01 +0300 Subject: [PATCH 045/194] couple changes in submitTesTask() method in TesTaskServiceImpl --- .../epam/pipeline/tesadapter/service/TesTaskServiceImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 0153aaffdf..3e0a8715a7 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -29,8 +29,7 @@ public TesCreateTaskResponse submitTesTask(TesTask body) { TesCreateTaskResponse tesCreateTaskResponse = new TesCreateTaskResponse(); PipelineRun pipelineRun = cloudPipelineAPIClient.runPipeline(taskMapper.mapToPipelineStart(body)); Assert.notNull(pipelineRun.getId(), "INVALID PIPELINE ID"); - String runId = pipelineRun.getId().toString(); - tesCreateTaskResponse.setId(runId); + tesCreateTaskResponse.setId("" + pipelineRun.getId()); return tesCreateTaskResponse; } From 397b3ee66e790e6332757774cdfdc403d4045e29 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Mon, 2 Sep 2019 14:32:56 +0300 Subject: [PATCH 046/194] couple clean-code for TaskMapper class. --- .../com/epam/pipeline/tesadapter/service/TaskMapper.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 2c669262af..a4348fe841 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -41,18 +41,13 @@ public PipelineStart mapToPipelineStart(TesTask tesTask) { pipelineStart.setIsSpot(tesTask.getResources().getPreemptible()); pipelineStart.setForce(false); pipelineStart.setNonPause(true); - - ListUtils.emptyIfNull(tesTask.getInputs()) - .forEach(tesInput -> + ListUtils.emptyIfNull(tesTask.getInputs()).forEach(tesInput -> params.put(tesInput.getName(), new PipeConfValueVO(tesInput.getUrl(), "input"))); - ListUtils.emptyIfNull(tesTask.getOutputs()).forEach(tesOutput -> params.put(tesOutput.getName(), new PipeConfValueVO(tesOutput.getUrl(), "output"))); - MapUtils.emptyIfNull(tesExecutor.getEnv()).forEach((name, value) -> params.put(name, new PipeConfValueVO(value, "string"))); pipelineStart.setParams(params); - return pipelineStart; } From 15460885c0e037ffabcc76816a3f8f8f24d8a1fe Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Mon, 2 Sep 2019 21:07:10 +0300 Subject: [PATCH 047/194] implemented Method GET /v1/tasks/{id} --- .../controller/TesAdapterController.java | 4 +- .../tesadapter/service/TaskMapper.java | 106 ++++++++++++++++++ .../tesadapter/service/TesTaskService.java | 3 + .../service/TesTaskServiceImpl.java | 11 +- 4 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index cfdafdddf0..83afe18857 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -53,8 +53,8 @@ ResponseEntity submitTesTask(@RequestBody TesTask body) { @ResponseBody ResponseEntity getTesTask(@PathVariable String id, @RequestParam(required = false, defaultValue = "MINIMAL") TaskView view) { - tesTaskService.stub(); - return new ResponseEntity(new TesTask(), HttpStatus.NOT_IMPLEMENTED); + return ResponseEntity.ok().body(tesTaskService.getTesTask(new Long(id))); + } @PostMapping("/v1/tasks/{id}:cancel") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java new file mode 100644 index 0000000000..546310ed81 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -0,0 +1,106 @@ +package com.epam.pipeline.tesadapter.service; + +import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.entity.pipeline.TaskStatus; +import com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter; +import com.epam.pipeline.tesadapter.entity.TesExecutor; +import com.epam.pipeline.tesadapter.entity.TesInput; +import com.epam.pipeline.tesadapter.entity.TesOutput; +import com.epam.pipeline.tesadapter.entity.TesResources; +import com.epam.pipeline.tesadapter.entity.TesState; +import com.epam.pipeline.tesadapter.entity.TesTask; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.Predicate; + +@Service +public class TaskMapper { + public TesTask mapToTesTask(PipelineRun run) { + TesTask tesTask = new TesTask(); + tesTask.setId(String.valueOf(run.getId())); + tesTask.setState(createTesState(run.getStatus())); + tesTask.setName(run.getPodId()); + tesTask.setDescription(""); + tesTask.setResources(createTesResources(run)); + tesTask.setExecutors(createListExecutor(run)); +// tesTask.setVolumes(); skip +// tesTask.setTags(); skip +// tesTask.setLogs(); skip + tesTask.setInputs(createTesInput(run.getPipelineRunParameters())); + tesTask.setOutputs(createTesOutput(run.getPipelineRunParameters())); + tesTask.setCreationTime(run.getStartDate().toString()); + return tesTask; + } + + private TesState createTesState(TaskStatus status){ + TesState tesStatus = null; + switch (status){ + case RUNNING: + tesStatus = TesState.RUNNING; + break; + case PAUSED: + tesStatus = TesState.PAUSED; + break; + case SUCCESS: + tesStatus = TesState.COMPLETE; + break; + case FAILURE: + tesStatus = TesState.EXECUTOR_ERROR; + break; + case STOPPED: + tesStatus = TesState.CANCELED; + break; + } + return tesStatus; + } + + private List createTesInput(List parameters){ + List listTesInput = new ArrayList<>(); + Predicate predicate = (s -> s.getType().contains("input")); + parameters.stream().filter(predicate).forEach(pipelineRunParameter -> { + TesInput tesInput = new TesInput(); + tesInput.setName(pipelineRunParameter.getName()); + tesInput.setUrl(pipelineRunParameter.getValue()); + listTesInput.add(tesInput); + }); + return listTesInput; + } + + private List createTesOutput(List parameters){ + List listTesOutput = new ArrayList<>(); + Predicate pipelineRunParameterPredicate = (s) -> s.getType().equals("output"); + parameters.stream().filter(pipelineRunParameterPredicate).forEach(pipelineRunParameter -> { + TesOutput output = new TesOutput(); + output.setName(pipelineRunParameter.getName()); + output.setUrl(pipelineRunParameter.getValue()); + listTesOutput.add(output); + }); + return listTesOutput; + } + + private TesResources createTesResources(PipelineRun run){ + TesResources tesResources = new TesResources(); +// tesResources.setCpuCores(); skip, later get from instance.nodeType + tesResources.setPreemptible(run.getInstance().getSpot()); +// tesResources.setRamGb(); skip, later get from instance.nodeType + tesResources.setDiskGb(new Double(run.getInstance().getNodeDisk())); +// tesResources.setZones(); skip, later get from region + return tesResources; + } + + private List createListExecutor(PipelineRun run){ + List tesExecutorList = new ArrayList<>(); + TesExecutor tesExecutor = new TesExecutor(); + tesExecutor.setCommand(new ArrayList(Arrays.asList(run.getActualCmd().split(" ")))); + tesExecutor.setWorkdir(""); + tesExecutor.setStdin(""); + tesExecutor.setStdout(""); + tesExecutor.setStderr(""); + tesExecutor.setEnv(run.getEnvVars()); + tesExecutorList.add(tesExecutor); + return tesExecutorList; + } +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index 81e69b9b86..67df172292 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -2,6 +2,7 @@ import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import com.epam.pipeline.tesadapter.entity.TesTask; public interface TesTaskService { @@ -9,4 +10,6 @@ public interface TesTaskService { void stub(); TesCancelTaskResponse cancelTesTask(String id); + + TesTask getTesTask(Long id); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index e00b760dcf..04f9be806d 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -3,6 +3,7 @@ import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.vo.RunStatusVO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -13,10 +14,12 @@ @Service public class TesTaskServiceImpl implements TesTaskService { private final CloudPipelineAPIClient cloudPipelineAPIClient; + private TaskMapper taskMapper; @Autowired - public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient) { + public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient, TaskMapper taskMapper) { this.cloudPipelineAPIClient = cloudPipelineAPIClient; + this.taskMapper = taskMapper; } @Override @@ -37,6 +40,12 @@ public TesCancelTaskResponse cancelTesTask(String id) { return new TesCancelTaskResponse(); } + @Override + public TesTask getTesTask(Long id) { + return taskMapper.mapToTesTask(cloudPipelineAPIClient.loadPipelineRun(id)); + + } + private Long parseRunId(String id) { Assert.hasText(id, "INVALID RUN ID"); try { From 87754c6e306765ac043a419cb92f0c152b97ff38 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 3 Sep 2019 16:51:21 +0300 Subject: [PATCH 048/194] updated current state of working submitTesTask() method. Added MessageHelper to resolve error, debug etc cases with message template. Fixed "magic numbers", and some little mistakes. Merged with "f_tes_web_mvc_test_for_submitTask" branch --- .../tesadapter/common/MessageConstants.java | 12 +++++ .../tesadapter/common/MessageHelper.java | 54 +++++++++++++++++++ .../configuration/RestConfiguration.java | 15 ++++++ .../tesadapter/service/TaskMapper.java | 38 ++++++++----- .../service/TesTaskServiceImpl.java | 23 ++++++-- .../src/main/resources/message.properties | 8 +++ .../app/TesAdapterControllerTest.java | 32 ++++++++--- 7 files changed, 156 insertions(+), 26 deletions(-) create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageHelper.java create mode 100644 tes-adapter/src/main/resources/message.properties diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java new file mode 100644 index 0000000000..9114df689e --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java @@ -0,0 +1,12 @@ +package com.epam.pipeline.tesadapter.common; + +public final class MessageConstants { + + //Common errors + public static final String ERROR_PARAMETER_REQUIRED = "error.parameter.required"; + public static final String ERROR_PARAMETER_NULL_OR_EMPTY = "error.null.param"; + public static final String ERROR_PARAMETER_INCOMPATIBLE_CONTENT = "error.parameter.incompatible.content"; + + //Parameters mapping + public static final String ERROR_PARAMETER_NON_SCALAR_TYPE = "error.parameter.non.scalar.type"; +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageHelper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageHelper.java new file mode 100644 index 0000000000..efc17b2ee6 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageHelper.java @@ -0,0 +1,54 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.tesadapter.common; + +import org.springframework.context.MessageSource; +import org.springframework.stereotype.Component; + +import java.util.Locale; + +@Component +public class MessageHelper { + + private MessageSource messageSource; + + public MessageHelper(final MessageSource messageSource) { + this.messageSource = messageSource; + } + + public MessageSource getMessageSource() { + return messageSource; + } + + /** + * Tries to resolve the message. Returns the resolved and formatted message or the given message code, when + * no appropriate message was found in available resources. + * + * @param code {@code String} represents the code to look up + * @param args represents a reference on array of {@code Object} or varargs; both provide arguments that will + * be filled in for params within message, or null if no arguments; args look like "{0}", + * "{1, date}", "{2, time}" within message (also see e.g. Spring documentation for more details) + * @return {@code String} represents the resolved message if the lookup was successful; otherwise the given + * keycode + * @see java.text.MessageFormat + * @see org.springframework.context.support.AbstractMessageSource + */ + public String getMessage(final String code, final Object... args) { + return getMessageSource().getMessage(code, args, code, Locale.getDefault()); + } + +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java index fa54d6c4b2..8a30e5ecf3 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java @@ -1,8 +1,10 @@ package com.epam.pipeline.tesadapter.configuration; +import com.epam.pipeline.tesadapter.common.MessageHelper; import org.springframework.boot.autoconfigure.http.HttpMessageConverters; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.support.ResourceBundleMessageSource; import org.springframework.http.converter.HttpMessageConverter; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; @@ -17,4 +19,17 @@ public HttpMessageConverters customConverters() { HttpMessageConverter addition = new MappingJackson2HttpMessageConverter(); return new HttpMessageConverters(false, Collections.singletonList(addition)); } + + @Bean + public MessageHelper messageHelper() { + return new MessageHelper(messageSource()); + } + + @Bean + public ResourceBundleMessageSource messageSource() { + ResourceBundleMessageSource messageSource = new ResourceBundleMessageSource(); + messageSource.setBasename("messages"); + messageSource.setDefaultEncoding("UTF-8"); + return messageSource; + } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index a4348fe841..fd2b9650f8 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -3,25 +3,38 @@ import com.epam.pipeline.entity.configuration.ExecutionEnvironment; import com.epam.pipeline.entity.configuration.PipeConfValueVO; import com.epam.pipeline.entity.pipeline.run.PipelineStart; +import com.epam.pipeline.tesadapter.common.MessageConstants; +import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.entity.TesExecutor; import com.epam.pipeline.tesadapter.entity.TesTask; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.ListUtils; import org.apache.commons.collections4.MapUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; +import org.springframework.stereotype.Service; +import org.springframework.util.Assert; import java.util.HashMap; import java.util.List; import java.util.Map; @Slf4j -@Component +@Service public class TaskMapper { private final String defaultInstanceType; + private final Integer defaultHddSize; + @Autowired + private MessageHelper messageHelper; + + private static final String SEPARATOR = " "; + private static final String INPUT_TYPE = "input"; + private static final String OUTPUT_TYPE = "output"; + private static final String DEFAULT_TYPE = "string"; + private static final Integer FIRST = 0; + public TaskMapper(@Value("${cloud.pipeline.instanceType}") String instanceType, @Value("${cloud.pipeline.hddSize}") Integer hddSize) { this.defaultInstanceType = instanceType; @@ -31,9 +44,9 @@ public TaskMapper(@Value("${cloud.pipeline.instanceType}") String instanceType, public PipelineStart mapToPipelineStart(TesTask tesTask) { PipelineStart pipelineStart = new PipelineStart(); Map params = new HashMap<>(); - TesExecutor tesExecutor = getExecutorFromTesExecutorsList(ListUtils.emptyIfNull(tesTask.getExecutors())); + TesExecutor tesExecutor = getExecutorFromTesExecutorsList(tesTask.getExecutors()); - pipelineStart.setCmdTemplate(String.join(" ", tesExecutor.getCommand())); + pipelineStart.setCmdTemplate(String.join(SEPARATOR, tesExecutor.getCommand())); pipelineStart.setDockerImage(tesExecutor.getImage()); pipelineStart.setExecutionEnvironment(ExecutionEnvironment.CLOUD_PLATFORM); pipelineStart.setHddSize(defaultHddSize); @@ -42,21 +55,18 @@ public PipelineStart mapToPipelineStart(TesTask tesTask) { pipelineStart.setForce(false); pipelineStart.setNonPause(true); ListUtils.emptyIfNull(tesTask.getInputs()).forEach(tesInput -> - params.put(tesInput.getName(), new PipeConfValueVO(tesInput.getUrl(), "input"))); + params.put(tesInput.getName(), new PipeConfValueVO(tesInput.getUrl(), INPUT_TYPE))); ListUtils.emptyIfNull(tesTask.getOutputs()).forEach(tesOutput -> - params.put(tesOutput.getName(), new PipeConfValueVO(tesOutput.getUrl(), "output"))); + params.put(tesOutput.getName(), new PipeConfValueVO(tesOutput.getUrl(), OUTPUT_TYPE))); MapUtils.emptyIfNull(tesExecutor.getEnv()).forEach((name, value) -> - params.put(name, new PipeConfValueVO(value, "string"))); + params.put(name, new PipeConfValueVO(value, DEFAULT_TYPE))); pipelineStart.setParams(params); return pipelineStart; } private TesExecutor getExecutorFromTesExecutorsList(List tesExecutors) { - if (CollectionUtils.isEmpty(tesExecutors)) { - log.error("LIST OF EXECUTORS EMPTY OR NULL"); - throw new IllegalArgumentException(); - } else { - return tesExecutors.get(0); - } + Assert.notEmpty(tesExecutors, messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, + "executors")); + return tesExecutors.get(FIRST); } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 3e0a8715a7..0bb1bf5726 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -1,7 +1,10 @@ package com.epam.pipeline.tesadapter.service; + import com.epam.pipeline.entity.pipeline.PipelineRun; import com.epam.pipeline.entity.pipeline.TaskStatus; +import com.epam.pipeline.tesadapter.common.MessageConstants; +import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; @@ -16,20 +19,28 @@ @Service public class TesTaskServiceImpl implements TesTaskService { private final CloudPipelineAPIClient cloudPipelineAPIClient; + private final TaskMapper taskMapper; + private final MessageHelper messageHelper; + + private final static String ID = "id"; + @Autowired - public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient, TaskMapper taskMapper) { + public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient, TaskMapper taskMapper, + MessageHelper messageHelper) { this.cloudPipelineAPIClient = cloudPipelineAPIClient; this.taskMapper = taskMapper; + this.messageHelper = messageHelper; } @Override public TesCreateTaskResponse submitTesTask(TesTask body) { TesCreateTaskResponse tesCreateTaskResponse = new TesCreateTaskResponse(); PipelineRun pipelineRun = cloudPipelineAPIClient.runPipeline(taskMapper.mapToPipelineStart(body)); - Assert.notNull(pipelineRun.getId(), "INVALID PIPELINE ID"); - tesCreateTaskResponse.setId("" + pipelineRun.getId()); + Assert.notNull(pipelineRun.getId(), messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_REQUIRED, + ID, TesCreateTaskResponse.class.getSimpleName())); + tesCreateTaskResponse.setId(String.valueOf(pipelineRun.getId())); return tesCreateTaskResponse; } @@ -52,11 +63,13 @@ public TesCancelTaskResponse cancelTesTask(String id) { } private Long parseRunId(String id) { - Assert.hasText(id, "INVALID RUN ID"); + Assert.hasText(id, messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_NON_SCALAR_TYPE, + ID, id)); try { return Long.parseLong(id); } catch (NumberFormatException e) { - log.error("INVALID RUN ID"); + log.error(messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, + ID, id)); throw new IllegalArgumentException(e); } } diff --git a/tes-adapter/src/main/resources/message.properties b/tes-adapter/src/main/resources/message.properties new file mode 100644 index 0000000000..a43b1aec2b --- /dev/null +++ b/tes-adapter/src/main/resources/message.properties @@ -0,0 +1,8 @@ +#general error messages +error.null.param=Error: the required parameter ''{0}'' is null or empty +error.parameter.required=Parameter ''{0}'' id required for ''{1}'' podId. + + +#Parameters +error.parameter.non.scalar.type=Unable to resolve parameter value ''{0}''. Reference value for field ''{1}'' is used as a parameter value. +error.parameter.incompatible.content=Parameter (id ''{0}'') is not compatible with content ''{1}''. \ No newline at end of file diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 70a8fd56eb..3106db2364 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -3,7 +3,9 @@ import com.epam.pipeline.tesadapter.controller.TesAdapterController; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; +import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.tesadapter.service.CloudPipelineAPIClient; import com.epam.pipeline.tesadapter.service.TesTaskServiceImpl; import org.junit.jupiter.api.Test; @@ -11,22 +13,28 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.servlet.MockMvc; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; + import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @ExtendWith(SpringExtension.class) @WebMvcTest(TesAdapterController.class) -@SuppressWarnings("unused") +@SuppressWarnings({"unused", "PMD.TooManyStaticImports"}) public class TesAdapterControllerTest { private static final String STUBBED_TASK_ID = "5"; private static final Long STUBBED_PAGE_SIZE = 55L; + private static final String STUBBED_SUBMIT_JSON_REQUEST = "{}"; + private static final String STUBBED_SUBMIT_JSON_RESPONSE = "{\"id\":\"5\"}"; @Autowired private MockMvc mockMvc; @@ -38,16 +46,26 @@ public class TesAdapterControllerTest { private CloudPipelineAPIClient cloudPipelineAPIClient; @Test - public void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { + void submitTesTaskWhenRequestingTesTaskBodyAndReturnId() throws Exception { + TesCreateTaskResponse tesCreateTaskResponse = new TesCreateTaskResponse(); + tesCreateTaskResponse.setId(STUBBED_TASK_ID); + when(tesTaskService.submitTesTask(any(TesTask.class))).thenReturn(tesCreateTaskResponse); + this.mockMvc.perform(post("/v1/tasks").contentType(MediaType.APPLICATION_JSON_UTF8) + .content(STUBBED_SUBMIT_JSON_REQUEST)) + .andDo(print()).andExpect(status().isOk()).andExpect(content().json(STUBBED_SUBMIT_JSON_RESPONSE)); + } + + @Test + void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { when(tesTaskService.cancelTesTask(STUBBED_TASK_ID)).thenReturn(new TesCancelTaskResponse()); - this.mockMvc.perform(post("/v1/tasks/{id}:cancel", STUBBED_TASK_ID)).andDo(print()) - .andExpect(status().isOk()); + this.mockMvc.perform(post("/v1/tasks/{id}:cancel", STUBBED_TASK_ID)) + .andDo(print()).andExpect(status().isOk()); } @Test - public void listTesTaskWhenRequestingReturnTesListTasksResponse() throws Exception { + void listTesTaskWhenRequestingReturnTesListTasksResponse() throws Exception { when(tesTaskService.listTesTask()).thenReturn(new TesListTasksResponse()); - this.mockMvc.perform(get("/v1/tasks?page_size={size}", STUBBED_PAGE_SIZE)).andDo(print()) - .andExpect(status().isOk()); + this.mockMvc.perform(get("/v1/tasks?page_size={size}", STUBBED_PAGE_SIZE)) + .andDo(print()).andExpect(status().isOk()); } } From c548d662ef287f71bcb932912321d72ebe1c7d8a Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 3 Sep 2019 18:02:05 +0300 Subject: [PATCH 049/194] refactoring tesTaskServiceImpl used Java 8 methods --- .../tesadapter/service/TesTaskService.java | 2 +- .../service/TesTaskServiceImpl.java | 23 ++++++++++--------- .../src/main/resources/application.properties | 4 +++- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index 131701fae3..3059b5bb12 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -11,5 +11,5 @@ public interface TesTaskService { TesCancelTaskResponse cancelTesTask(String id); - TesServiceInfo getServiceInfo(); + TesServiceInfo getServiceInfo(String nameOfService, String doc); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 3a89627ac6..e713cb7fcc 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -7,12 +7,15 @@ import com.epam.pipeline.tesadapter.entity.TesServiceInfo; import com.epam.pipeline.vo.RunStatusVO; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import java.util.ArrayList; import java.util.List; +import java.util.stream.Stream; @Slf4j @Service @@ -53,22 +56,20 @@ private Long parseRunId(String id) { } @Override - public TesServiceInfo getServiceInfo() { - String nameOfService = "CloudPipeline"; - String doc = "https://epam.github.io/cloud-pipeline/"; - TesServiceInfo tesServiceInfo = new TesServiceInfo(); - tesServiceInfo.setName(nameOfService); - tesServiceInfo.setDoc(doc); - tesServiceInfo.setStorage(getDataStorage()); - return tesServiceInfo; + public TesServiceInfo getServiceInfo(@Value("${TesTaskServiceImpl.nameOfService}") String nameOfService, + @Value("${TesTaskServiceImpl.doc}") String doc) { + Stream tesServiceInfoStream = Stream.of(new TesServiceInfo()) + .peek(t -> t.setName(nameOfService)) + .peek(t -> t.setDoc(doc)) + .peek(t -> t.setStorage(getDataStorage())); + return tesServiceInfoStream.findFirst().get(); } private List getDataStorage(){ List listPathDataStorage = new ArrayList<>(); List storageList = cloudPipelineAPIClient.loadAllDataStorages(); - for (AbstractDataStorage i: storageList){ - listPathDataStorage.add(i.getPath()); - } + ListUtils.emptyIfNull(storageList); + storageList.stream().forEach(storage -> listPathDataStorage.add(storage.getPath())); return listPathDataStorage; } } diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index 1c344ce654..ee8ce3b98f 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -1,3 +1,5 @@ server.port=8080 cloud.pipeline.host=${API} -cloud.pipeline.token=${API_TOKEN} \ No newline at end of file +cloud.pipeline.token=${API_TOKEN} +TesTaskServiceImpl.nameOfService=${CloudPipeline} +TesTaskServiceImpl.doc=${https://epam.github.io/cloud-pipeline/} \ No newline at end of file From 55dac88d27170a5df3168b83dcb6e46a77dfe419 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 3 Sep 2019 18:11:11 +0300 Subject: [PATCH 050/194] added separate AppConfiguration class for MessageHelper, removed @ResponseBody for TesAdapterController class --- .../configuration/AppConfiguration.java | 23 +++++++++++++++++++ .../configuration/RestConfiguration.java | 15 ------------ .../controller/TesAdapterController.java | 4 ---- 3 files changed, 23 insertions(+), 19 deletions(-) create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/AppConfiguration.java diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/AppConfiguration.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/AppConfiguration.java new file mode 100644 index 0000000000..e9a702a201 --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/AppConfiguration.java @@ -0,0 +1,23 @@ +package com.epam.pipeline.tesadapter.configuration; + +import com.epam.pipeline.tesadapter.common.MessageHelper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.support.ResourceBundleMessageSource; + +@Configuration +public class AppConfiguration { + + @Bean + public MessageHelper messageHelper() { + return new MessageHelper(messageSource()); + } + + @Bean + public ResourceBundleMessageSource messageSource() { + ResourceBundleMessageSource messageSource = new ResourceBundleMessageSource(); + messageSource.setBasename("messages"); + messageSource.setDefaultEncoding("UTF-8"); + return messageSource; + } +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java index 8a30e5ecf3..fa54d6c4b2 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java @@ -1,10 +1,8 @@ package com.epam.pipeline.tesadapter.configuration; -import com.epam.pipeline.tesadapter.common.MessageHelper; import org.springframework.boot.autoconfigure.http.HttpMessageConverters; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.support.ResourceBundleMessageSource; import org.springframework.http.converter.HttpMessageConverter; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; @@ -19,17 +17,4 @@ public HttpMessageConverters customConverters() { HttpMessageConverter addition = new MappingJackson2HttpMessageConverter(); return new HttpMessageConverters(false, Collections.singletonList(addition)); } - - @Bean - public MessageHelper messageHelper() { - return new MessageHelper(messageSource()); - } - - @Bean - public ResourceBundleMessageSource messageSource() { - ResourceBundleMessageSource messageSource = new ResourceBundleMessageSource(); - messageSource.setBasename("messages"); - messageSource.setDefaultEncoding("UTF-8"); - return messageSource; - } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index b48a8155f7..370392f3fc 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -14,7 +14,6 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; @RestController @@ -43,13 +42,11 @@ public ResponseEntity listTesTasks( } @PostMapping("/v1/tasks") - @ResponseBody ResponseEntity submitTesTask(@RequestBody TesTask body) { return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.submitTesTask(body)); } @GetMapping("/v1/tasks/{id}") - @ResponseBody ResponseEntity getTesTask(@PathVariable String id, @RequestParam(required = false, defaultValue = "MINIMAL") TaskView view) { tesTaskService.stub(); @@ -57,7 +54,6 @@ ResponseEntity getTesTask(@PathVariable String id, @RequestParam(requir } @PostMapping("/v1/tasks/{id}:cancel") - @ResponseBody ResponseEntity cancelTesTask(@PathVariable String id) { return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.cancelTesTask(id)); } From 05c68224e966dc72214ea1f7d7edc02848e61f82 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 3 Sep 2019 18:28:04 +0300 Subject: [PATCH 051/194] refactoring build.gradle --- tes-adapter/build.gradle | 9 --------- .../tesadapter/service/TesTaskServiceImplTest.java | 7 ++++--- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index 6833babb44..17498e388c 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -66,13 +66,4 @@ dependencies { testImplementation('org.springframework.boot:spring-boot-starter-test') { exclude group: 'junit', module: 'junit' //direct exclusion junit4 } - testImplementation('org.junit.jupiter:junit-jupiter-api:5.2.0') - testCompile('org.junit.jupiter:junit-jupiter-params:5.2.0') - testRuntime('org.junit.jupiter:junit-jupiter-engine:5.2.0') - testCompile "org.mockito:mockito-core:2.+" - testCompile('org.mockito:mockito-junit-jupiter:2.18.3') - testCompile('org.springframework.boot:spring-boot-starter-test') - testImplementation('org.springframework.boot:spring-boot-starter-test') { - exclude group: 'junit', module: 'junit' - } } \ No newline at end of file diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java index a62eb44b51..2c64023c78 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java @@ -1,7 +1,8 @@ package com.epam.pipeline.tesadapter.service; + import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -9,8 +10,8 @@ class TesTaskServiceImplTest { private CloudPipelineAPIClient cloudPipelineAPIClient; private TesTaskServiceImpl tesTaskService; - @BeforeAll - void setUp() { + @BeforeEach + public void setUp() { cloudPipelineAPIClient = Mockito.mock(CloudPipelineAPIClient.class); tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient); } From 3f954fd0fed6dfb784c5e385c62d46bca0f9ce5a Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 3 Sep 2019 19:45:42 +0300 Subject: [PATCH 052/194] add TaskMapper without test --- .../tesadapter/service/TaskMapper.java | 62 ++++++++++--------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 546310ed81..44d0494b6f 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -9,12 +9,14 @@ import com.epam.pipeline.tesadapter.entity.TesResources; import com.epam.pipeline.tesadapter.entity.TesState; import com.epam.pipeline.tesadapter.entity.TesTask; +import org.apache.commons.collections4.ListUtils; import org.springframework.stereotype.Service; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Service public class TaskMapper { @@ -29,8 +31,8 @@ public TesTask mapToTesTask(PipelineRun run) { // tesTask.setVolumes(); skip // tesTask.setTags(); skip // tesTask.setLogs(); skip - tesTask.setInputs(createTesInput(run.getPipelineRunParameters())); - tesTask.setOutputs(createTesOutput(run.getPipelineRunParameters())); + tesTask.setInputs(createTesInput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))); + tesTask.setOutputs(createTesOutput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))); tesTask.setCreationTime(run.getStartDate().toString()); return tesTask; } @@ -58,49 +60,53 @@ private TesState createTesState(TaskStatus status){ } private List createTesInput(List parameters){ - List listTesInput = new ArrayList<>(); - Predicate predicate = (s -> s.getType().contains("input")); - parameters.stream().filter(predicate).forEach(pipelineRunParameter -> { - TesInput tesInput = new TesInput(); - tesInput.setName(pipelineRunParameter.getName()); - tesInput.setUrl(pipelineRunParameter.getValue()); - listTesInput.add(tesInput); - }); - return listTesInput; + TesInput tesInput = new TesInput(); + Stream stream = Stream.of(tesInput) + .peek(i -> { + Predicate predicate = (s -> s.getType().contains("input")); + Stream pipelineRunParameterStream = parameters.stream() + .filter(predicate) + .peek(pipelineRunParameter -> { + tesInput.setName(pipelineRunParameter.getName()); + tesInput.setUrl(pipelineRunParameter.getValue()); + }); + }); + return stream.collect(Collectors.toList()); } private List createTesOutput(List parameters){ - List listTesOutput = new ArrayList<>(); - Predicate pipelineRunParameterPredicate = (s) -> s.getType().equals("output"); - parameters.stream().filter(pipelineRunParameterPredicate).forEach(pipelineRunParameter -> { - TesOutput output = new TesOutput(); - output.setName(pipelineRunParameter.getName()); - output.setUrl(pipelineRunParameter.getValue()); - listTesOutput.add(output); - }); - return listTesOutput; + TesOutput tesOutput = new TesOutput(); + Stream stream = Stream.of(tesOutput) + .peek(i -> { + Predicate predicate = (s -> s.getType().contains("output")); + Stream pipelineRunParameterStream = parameters.stream() + .filter(predicate) + .peek(pipelineRunParameter -> { + tesOutput.setName(pipelineRunParameter.getName()); + tesOutput.setUrl(pipelineRunParameter.getValue()); + }); + }); + return stream.collect(Collectors.toList()); } private TesResources createTesResources(PipelineRun run){ TesResources tesResources = new TesResources(); // tesResources.setCpuCores(); skip, later get from instance.nodeType - tesResources.setPreemptible(run.getInstance().getSpot()); + tesResources.setPreemptible(run.getInstance().getSpot()); // tesResources.setRamGb(); skip, later get from instance.nodeType - tesResources.setDiskGb(new Double(run.getInstance().getNodeDisk())); + tesResources.setDiskGb(new Double(run.getInstance().getNodeDisk())); // tesResources.setZones(); skip, later get from region - return tesResources; + return tesResources; } private List createListExecutor(PipelineRun run){ - List tesExecutorList = new ArrayList<>(); TesExecutor tesExecutor = new TesExecutor(); - tesExecutor.setCommand(new ArrayList(Arrays.asList(run.getActualCmd().split(" ")))); + tesExecutor.setCommand(Arrays.asList(run.getActualCmd().split(" "))); tesExecutor.setWorkdir(""); tesExecutor.setStdin(""); tesExecutor.setStdout(""); tesExecutor.setStderr(""); tesExecutor.setEnv(run.getEnvVars()); - tesExecutorList.add(tesExecutor); - return tesExecutorList; + return Stream.of(tesExecutor).collect(Collectors.toList()); } } From 07d19a08e7f1dad3d1842b7de2fc1b003846a1ef Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 4 Sep 2019 12:06:12 +0300 Subject: [PATCH 053/194] refactoring TesTaskServisImpl() --- .../controller/TesAdapterController.java | 1 - .../tesadapter/service/TesTaskService.java | 2 +- .../service/TesTaskServiceImpl.java | 19 ++++++++++++------- .../src/main/resources/application.properties | 4 ++-- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index a18b36e606..93fdb77983 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -20,7 +20,6 @@ @RestController public class TesAdapterController { - private TesTaskService tesTaskService; @Autowired diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index 3059b5bb12..131701fae3 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -11,5 +11,5 @@ public interface TesTaskService { TesCancelTaskResponse cancelTesTask(String id); - TesServiceInfo getServiceInfo(String nameOfService, String doc); + TesServiceInfo getServiceInfo(); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index e713cb7fcc..03fd4d5cd6 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -10,16 +10,24 @@ import org.apache.commons.collections4.ListUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.PropertySource; import org.springframework.stereotype.Service; import org.springframework.util.Assert; -import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import java.util.stream.Stream; @Slf4j @Service +@PropertySource("application.properties") public class TesTaskServiceImpl implements TesTaskService { + @Value("${cloud.pipeline.nameOfService}") + private String nameOfService; + + @Value("${cloud.pipeline.doc}") + private String doc; + private final CloudPipelineAPIClient cloudPipelineAPIClient; @Autowired @@ -56,8 +64,7 @@ private Long parseRunId(String id) { } @Override - public TesServiceInfo getServiceInfo(@Value("${TesTaskServiceImpl.nameOfService}") String nameOfService, - @Value("${TesTaskServiceImpl.doc}") String doc) { + public TesServiceInfo getServiceInfo() { Stream tesServiceInfoStream = Stream.of(new TesServiceInfo()) .peek(t -> t.setName(nameOfService)) .peek(t -> t.setDoc(doc)) @@ -65,11 +72,9 @@ public TesServiceInfo getServiceInfo(@Value("${TesTaskServiceImpl.nameOfService} return tesServiceInfoStream.findFirst().get(); } - private List getDataStorage(){ - List listPathDataStorage = new ArrayList<>(); + private List getDataStorage() { List storageList = cloudPipelineAPIClient.loadAllDataStorages(); ListUtils.emptyIfNull(storageList); - storageList.stream().forEach(storage -> listPathDataStorage.add(storage.getPath())); - return listPathDataStorage; + return storageList.stream().map(storage -> storage.getPath()).collect(Collectors.toList()); } } diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index ee8ce3b98f..a41549c532 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -1,5 +1,5 @@ server.port=8080 cloud.pipeline.host=${API} cloud.pipeline.token=${API_TOKEN} -TesTaskServiceImpl.nameOfService=${CloudPipeline} -TesTaskServiceImpl.doc=${https://epam.github.io/cloud-pipeline/} \ No newline at end of file +cloud.pipeline.nameOfService=${CloudPipeline} +cloud.pipeline.doc=${https://epam.github.io/cloud-pipeline/} \ No newline at end of file From 7850727028dfa7750840f2aadb1396f2e39d6e12 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 4 Sep 2019 17:55:53 +0300 Subject: [PATCH 054/194] refactoring TesTaskServisImpl() with java 8 style --- .../pipeline/tesadapter/service/TesTaskServiceImpl.java | 7 +++---- tes-adapter/src/main/resources/application.properties | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 03fd4d5cd6..84bc78b73f 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -22,10 +22,10 @@ @Service @PropertySource("application.properties") public class TesTaskServiceImpl implements TesTaskService { - @Value("${cloud.pipeline.nameOfService}") + @Value("cloud.pipeline.service.name") private String nameOfService; - @Value("${cloud.pipeline.doc}") + @Value("cloud.pipeline.doc") private String doc; private final CloudPipelineAPIClient cloudPipelineAPIClient; @@ -74,7 +74,6 @@ public TesServiceInfo getServiceInfo() { private List getDataStorage() { List storageList = cloudPipelineAPIClient.loadAllDataStorages(); - ListUtils.emptyIfNull(storageList); - return storageList.stream().map(storage -> storage.getPath()).collect(Collectors.toList()); + return ListUtils.emptyIfNull(storageList).stream().map(storage -> storage.getPath()).collect(Collectors.toList()); } } diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index a41549c532..9b7bd9e0e2 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -1,5 +1,5 @@ server.port=8080 cloud.pipeline.host=${API} cloud.pipeline.token=${API_TOKEN} -cloud.pipeline.nameOfService=${CloudPipeline} -cloud.pipeline.doc=${https://epam.github.io/cloud-pipeline/} \ No newline at end of file +cloud.pipeline.service.name=CloudPipeline +cloud.pipeline.doc=https://epam.github.io/cloud-pipeline/ \ No newline at end of file From c68d2bef3b6c22600a7c622f1df32c828b14f046 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 4 Sep 2019 18:03:29 +0300 Subject: [PATCH 055/194] refactoring TesTaskServisImpl --- .../tesadapter/service/TesTaskServiceImpl.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 84bc78b73f..c2b2dd6435 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -16,7 +16,6 @@ import java.util.List; import java.util.stream.Collectors; -import java.util.stream.Stream; @Slf4j @Service @@ -65,11 +64,11 @@ private Long parseRunId(String id) { @Override public TesServiceInfo getServiceInfo() { - Stream tesServiceInfoStream = Stream.of(new TesServiceInfo()) - .peek(t -> t.setName(nameOfService)) - .peek(t -> t.setDoc(doc)) - .peek(t -> t.setStorage(getDataStorage())); - return tesServiceInfoStream.findFirst().get(); + TesServiceInfo tesServiceInfo = new TesServiceInfo(); + tesServiceInfo.setName(nameOfService); + tesServiceInfo.setDoc(doc); + tesServiceInfo.setStorage(getDataStorage()); + return tesServiceInfo; } private List getDataStorage() { From be70665789af254b54eac78be7ce08b5fe8226db Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 4 Sep 2019 18:12:53 +0300 Subject: [PATCH 056/194] refactoring TaskMapper --- .../java/com/epam/pipeline/tesadapter/service/TaskMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 44d0494b6f..7d9a050890 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -107,6 +107,6 @@ private List createListExecutor(PipelineRun run){ tesExecutor.setStdout(""); tesExecutor.setStderr(""); tesExecutor.setEnv(run.getEnvVars()); - return Stream.of(tesExecutor).collect(Collectors.toList()); + return ListUtils.emptyIfNull(Arrays.asList(tesExecutor)); } } From 73edb68e6a375c718017ff10521d68f6c8b44e51 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 4 Sep 2019 18:21:57 +0300 Subject: [PATCH 057/194] deleted @PropertySource from TesTaskServiceImpl --- .../epam/pipeline/tesadapter/service/TesTaskServiceImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index c2b2dd6435..e802b09f61 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -10,7 +10,6 @@ import org.apache.commons.collections4.ListUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.PropertySource; import org.springframework.stereotype.Service; import org.springframework.util.Assert; @@ -19,7 +18,6 @@ @Slf4j @Service -@PropertySource("application.properties") public class TesTaskServiceImpl implements TesTaskService { @Value("cloud.pipeline.service.name") private String nameOfService; From ed32fa20261d0d9b40109076d6680f972834c452 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 4 Sep 2019 18:32:11 +0300 Subject: [PATCH 058/194] refactoring TesTaskServiceImpl --- .../epam/pipeline/tesadapter/service/TesTaskServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index e802b09f61..87be1ebdaf 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -19,10 +19,10 @@ @Slf4j @Service public class TesTaskServiceImpl implements TesTaskService { - @Value("cloud.pipeline.service.name") + @Value("${cloud.pipeline.service.name}") private String nameOfService; - @Value("cloud.pipeline.doc") + @Value("${cloud.pipeline.doc}") private String doc; private final CloudPipelineAPIClient cloudPipelineAPIClient; From 3789262e1e5df523bd727ff68d73a00db8f9d3ea Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 4 Sep 2019 18:46:51 +0300 Subject: [PATCH 059/194] refactoring TesTaskServiceImpl --- .../pipeline/tesadapter/service/TesTaskServiceImpl.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 87be1ebdaf..442063abdd 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -21,10 +21,8 @@ public class TesTaskServiceImpl implements TesTaskService { @Value("${cloud.pipeline.service.name}") private String nameOfService; - @Value("${cloud.pipeline.doc}") private String doc; - private final CloudPipelineAPIClient cloudPipelineAPIClient; @Autowired @@ -62,7 +60,7 @@ private Long parseRunId(String id) { @Override public TesServiceInfo getServiceInfo() { - TesServiceInfo tesServiceInfo = new TesServiceInfo(); + final TesServiceInfo tesServiceInfo = new TesServiceInfo(); tesServiceInfo.setName(nameOfService); tesServiceInfo.setDoc(doc); tesServiceInfo.setStorage(getDataStorage()); @@ -70,7 +68,8 @@ public TesServiceInfo getServiceInfo() { } private List getDataStorage() { - List storageList = cloudPipelineAPIClient.loadAllDataStorages(); - return ListUtils.emptyIfNull(storageList).stream().map(storage -> storage.getPath()).collect(Collectors.toList()); + return ListUtils.emptyIfNull(cloudPipelineAPIClient.loadAllDataStorages()) + .stream().map(storage -> storage.getPath()) + .collect(Collectors.toList()); } } From 856c51e0590ca8f77dda10443084dd06261a3e68 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 4 Sep 2019 20:02:32 +0300 Subject: [PATCH 060/194] deleted @ResponseBody from Controller.I Did methods more simply: createTesInput and createTesOutput and fixed some little things --- .../controller/TesAdapterController.java | 15 +++-- .../tesadapter/service/TaskMapper.java | 61 +++++++------------ 2 files changed, 30 insertions(+), 46 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index 83afe18857..fb27bd1f5c 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -14,12 +14,10 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; @RestController public class TesAdapterController { - private TesTaskService tesTaskService; @Autowired @@ -43,23 +41,28 @@ public ResponseEntity listTesTasks( } @PostMapping("/v1/tasks") - @ResponseBody ResponseEntity submitTesTask(@RequestBody TesTask body) { tesTaskService.stub(); return new ResponseEntity(new TesCreateTaskResponse(), HttpStatus.NOT_IMPLEMENTED); } @GetMapping("/v1/tasks/{id}") - @ResponseBody ResponseEntity getTesTask(@PathVariable String id, @RequestParam(required = false, defaultValue = "MINIMAL") TaskView view) { - return ResponseEntity.ok().body(tesTaskService.getTesTask(new Long(id))); + return ResponseEntity.ok().body(tesTaskService.getTesTask(parseStringIdToLongId(id))); } @PostMapping("/v1/tasks/{id}:cancel") - @ResponseBody ResponseEntity cancelTesTask(@PathVariable String id) { return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.cancelTesTask(id)); } + + private Long parseStringIdToLongId(String id){ + try { + return Long.parseLong(id); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(e); + } + } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 7d9a050890..50cc32e3f1 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -14,23 +14,19 @@ import java.util.Arrays; import java.util.List; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; @Service public class TaskMapper { + private static final String INPUT = "input"; + private static final String OUTPUT = "output"; + public TesTask mapToTesTask(PipelineRun run) { TesTask tesTask = new TesTask(); tesTask.setId(String.valueOf(run.getId())); tesTask.setState(createTesState(run.getStatus())); tesTask.setName(run.getPodId()); - tesTask.setDescription(""); tesTask.setResources(createTesResources(run)); tesTask.setExecutors(createListExecutor(run)); -// tesTask.setVolumes(); skip -// tesTask.setTags(); skip -// tesTask.setLogs(); skip tesTask.setInputs(createTesInput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))); tesTask.setOutputs(createTesOutput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))); tesTask.setCreationTime(run.getStartDate().toString()); @@ -38,7 +34,7 @@ public TesTask mapToTesTask(PipelineRun run) { } private TesState createTesState(TaskStatus status){ - TesState tesStatus = null; + TesState tesStatus = TesState.UNKNOWN; switch (status){ case RUNNING: tesStatus = TesState.RUNNING; @@ -60,52 +56,37 @@ private TesState createTesState(TaskStatus status){ } private List createTesInput(List parameters){ - TesInput tesInput = new TesInput(); - Stream stream = Stream.of(tesInput) - .peek(i -> { - Predicate predicate = (s -> s.getType().contains("input")); - Stream pipelineRunParameterStream = parameters.stream() - .filter(predicate) - .peek(pipelineRunParameter -> { - tesInput.setName(pipelineRunParameter.getName()); - tesInput.setUrl(pipelineRunParameter.getValue()); - }); + final TesInput tesInput = new TesInput(); + parameters.stream() + .filter(pipelineRunParameter -> pipelineRunParameter.getType().contains(INPUT)) + .peek(pipelineRunParameter -> { + tesInput.setName(pipelineRunParameter.getName()); + tesInput.setUrl(pipelineRunParameter.getValue()); }); - return stream.collect(Collectors.toList()); + return ListUtils.emptyIfNull(Arrays.asList(tesInput)); } private List createTesOutput(List parameters){ - TesOutput tesOutput = new TesOutput(); - Stream stream = Stream.of(tesOutput) - .peek(i -> { - Predicate predicate = (s -> s.getType().contains("output")); - Stream pipelineRunParameterStream = parameters.stream() - .filter(predicate) - .peek(pipelineRunParameter -> { - tesOutput.setName(pipelineRunParameter.getName()); - tesOutput.setUrl(pipelineRunParameter.getValue()); - }); + final TesOutput tesOutput = new TesOutput(); + parameters.stream() + .filter(pipelineRunParameter -> pipelineRunParameter.getType().contains(OUTPUT)) + .peek(pipelineRunParameter -> { + tesOutput.setName(pipelineRunParameter.getName()); + tesOutput.setUrl(pipelineRunParameter.getValue()); }); - return stream.collect(Collectors.toList()); + return ListUtils.emptyIfNull(Arrays.asList(tesOutput)); } private TesResources createTesResources(PipelineRun run){ - TesResources tesResources = new TesResources(); -// tesResources.setCpuCores(); skip, later get from instance.nodeType + final TesResources tesResources = new TesResources(); tesResources.setPreemptible(run.getInstance().getSpot()); -// tesResources.setRamGb(); skip, later get from instance.nodeType tesResources.setDiskGb(new Double(run.getInstance().getNodeDisk())); -// tesResources.setZones(); skip, later get from region return tesResources; } private List createListExecutor(PipelineRun run){ - TesExecutor tesExecutor = new TesExecutor(); - tesExecutor.setCommand(Arrays.asList(run.getActualCmd().split(" "))); - tesExecutor.setWorkdir(""); - tesExecutor.setStdin(""); - tesExecutor.setStdout(""); - tesExecutor.setStderr(""); + final TesExecutor tesExecutor = new TesExecutor(); + tesExecutor.setCommand(ListUtils.emptyIfNull(Arrays.asList(run.getActualCmd().split(" ")))); tesExecutor.setEnv(run.getEnvVars()); return ListUtils.emptyIfNull(Arrays.asList(tesExecutor)); } From a3f488e9b187be4d1da4a5884d121b3cfb207e8c Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 4 Sep 2019 20:55:27 +0300 Subject: [PATCH 061/194] i merged branch tes-support to f_2803. and did refactoring parseStringToLong with MessageHelper --- .../tesadapter/controller/TesAdapterController.java | 12 +++++++++++- .../epam/pipeline/tesadapter/service/TaskMapper.java | 12 ++++++++++-- .../pipeline/tesadapter/service/TesTaskService.java | 1 - .../tesadapter/service/TesTaskServiceImpl.java | 1 - 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index 21885761f0..87c3ab52b0 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -1,14 +1,18 @@ package com.epam.pipeline.tesadapter.controller; +import com.epam.pipeline.tesadapter.common.MessageConstants; +import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.entity.TaskView; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.tesadapter.service.TesTaskService; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.util.Assert; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; @@ -16,12 +20,15 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +@Slf4j @RestController public class TesAdapterController { private TesTaskService tesTaskService; + private MessageHelper messageHelper; @Autowired - public TesAdapterController(TesTaskService tesTaskService) { + public TesAdapterController(TesTaskService tesTaskService, MessageHelper messageHelper) { + this.messageHelper = messageHelper; this.tesTaskService = tesTaskService; } @@ -58,9 +65,12 @@ ResponseEntity cancelTesTask(@PathVariable String id) { } private Long parseStringIdToLongId(String id){ + Assert.hasText(id, messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_NON_SCALAR_TYPE)); try { return Long.parseLong(id); } catch (NumberFormatException e) { + log.error(messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, + "ID", id)); throw new IllegalArgumentException(e); } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index dda1c26adb..bdd2b8ba55 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -2,10 +2,17 @@ import com.epam.pipeline.entity.configuration.ExecutionEnvironment; import com.epam.pipeline.entity.configuration.PipeConfValueVO; +import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.entity.pipeline.run.PipelineStart; +import com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter; import com.epam.pipeline.tesadapter.common.MessageConstants; import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.entity.TesExecutor; +import com.epam.pipeline.tesadapter.entity.TesInput; +import com.epam.pipeline.tesadapter.entity.TesOutput; +import com.epam.pipeline.tesadapter.entity.TesResources; +import com.epam.pipeline.tesadapter.entity.TesState; import com.epam.pipeline.tesadapter.entity.TesTask; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.ListUtils; @@ -15,6 +22,7 @@ import org.springframework.stereotype.Service; import org.springframework.util.Assert; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -71,7 +79,7 @@ private TesExecutor getExecutorFromTesExecutorsList(List tesExecuto } public TesTask mapToTesTask(PipelineRun run) { - TesTask tesTask = new TesTask(); + final TesTask tesTask = new TesTask(); tesTask.setId(String.valueOf(run.getId())); tesTask.setState(createTesState(run.getStatus())); tesTask.setName(run.getPodId()); @@ -136,7 +144,7 @@ private TesResources createTesResources(PipelineRun run){ private List createListExecutor(PipelineRun run){ final TesExecutor tesExecutor = new TesExecutor(); - tesExecutor.setCommand(ListUtils.emptyIfNull(Arrays.asList(run.getActualCmd().split(" ")))); + tesExecutor.setCommand(ListUtils.emptyIfNull(Arrays.asList(run.getActualCmd().split(SEPARATOR)))); tesExecutor.setEnv(run.getEnvVars()); return ListUtils.emptyIfNull(Arrays.asList(tesExecutor)); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index 2bd182e8ed..9420a9b15b 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -3,7 +3,6 @@ import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; import com.epam.pipeline.tesadapter.entity.TesTask; -import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; public interface TesTaskService { diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index d474435896..9962e5248f 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -19,7 +19,6 @@ @Service public class TesTaskServiceImpl implements TesTaskService { private final CloudPipelineAPIClient cloudPipelineAPIClient; - private TaskMapper taskMapper; private final TaskMapper taskMapper; From 522a3280e7ee4512d6c63abd2b34068c86b6569a Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 4 Sep 2019 23:31:25 +0300 Subject: [PATCH 062/194] first not finished state of InstanceTypes mapping. Need correspond mapping between TesTask and loadAllowedInstanceAndPriceTypes() --- .../client/pipeline/CloudPipelineAPI.java | 10 +++ .../service/CloudPipelineAPIClient.java | 6 ++ .../tesadapter/service/TaskMapper.java | 63 ++++++++++++++++--- .../src/main/resources/application.properties | 3 +- .../service/TesTaskServiceImplTest.java | 7 ++- 5 files changed, 79 insertions(+), 10 deletions(-) diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java index 6850ee47a5..57648aeb6a 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java @@ -16,6 +16,7 @@ package com.epam.pipeline.client.pipeline; +import com.epam.pipeline.entity.cluster.AllowedInstanceAndPriceTypes; import com.epam.pipeline.entity.cluster.NodeInstance; import com.epam.pipeline.entity.configuration.RunConfiguration; import com.epam.pipeline.entity.datastorage.AbstractDataStorage; @@ -72,6 +73,10 @@ public interface CloudPipelineAPI { String TOOL_IDENTIFIER = "image"; String VERSION = "version"; String PATH = "path"; + String TOOL_ID = "toolId"; + String REGION_ID = "regionId"; + String SPOT = "spot"; + @POST("run") Call> runPipeline(@Body PipelineStart runVo); @@ -193,6 +198,11 @@ Call>> loadRepositoryContent(@Path(ID) Long id, @POST("cluster/node/filter") Call>> findNodes(@Body FilterNodesVO filterNodesVO); + @GET("cluster/instance/allowed") + Call> loadAllowedInstanceAndPriceTypes(@Query(TOOL_ID) Long toolId, + @Query(REGION_ID) Long regionId, + @Query(SPOT) Boolean spot); + //Notification methods @POST("notification/message") Call> createNotification(@Body NotificationMessageVO notification); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index b707a593a2..ead7aac3f7 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -2,6 +2,7 @@ import com.epam.pipeline.client.pipeline.CloudPipelineAPI; import com.epam.pipeline.client.pipeline.CloudPipelineApiBuilder; +import com.epam.pipeline.entity.cluster.AllowedInstanceAndPriceTypes; import com.epam.pipeline.entity.pipeline.PipelineRun; import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.utils.QueryUtils; @@ -31,4 +32,9 @@ public PipelineRun loadPipelineRun(final Long pipelineRunId) { public PipelineRun updateRunStatus(final Long pipelineRunId, RunStatusVO statusUpdate) { return QueryUtils.execute(cloudPipelineAPI.updateRunStatus(pipelineRunId, statusUpdate)); } + + public AllowedInstanceAndPriceTypes loadAllowedInstanceAndPriceTypes(final Long toolId, final Long regionId, + final Boolean spot) { + return QueryUtils.execute(cloudPipelineAPI.loadAllowedInstanceAndPriceTypes(toolId, regionId, spot)); + } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index fd2b9650f8..bb5259c62c 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -1,5 +1,6 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.entity.cluster.AllowedInstanceAndPriceTypes; import com.epam.pipeline.entity.configuration.ExecutionEnvironment; import com.epam.pipeline.entity.configuration.PipeConfValueVO; import com.epam.pipeline.entity.pipeline.run.PipelineStart; @@ -15,6 +16,8 @@ import org.springframework.stereotype.Service; import org.springframework.util.Assert; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -22,12 +25,11 @@ @Slf4j @Service public class TaskMapper { - private final String defaultInstanceType; - private final Integer defaultHddSize; - - @Autowired + private final Long defaultToolId; + private final Long defaultRegionId; private MessageHelper messageHelper; + private final CloudPipelineAPIClient cloudPipelineAPIClient; private static final String SEPARATOR = " "; private static final String INPUT_TYPE = "input"; @@ -35,10 +37,17 @@ public class TaskMapper { private static final String DEFAULT_TYPE = "string"; private static final Integer FIRST = 0; - public TaskMapper(@Value("${cloud.pipeline.instanceType}") String instanceType, - @Value("${cloud.pipeline.hddSize}") Integer hddSize) { - this.defaultInstanceType = instanceType; + @Autowired + public TaskMapper(@Value("${cloud.pipeline.hddSize}") Integer hddSize, + @Value("${cloud.pipeline.instanceType.toolId}") Long defaultToolId, + @Value("${cloud.pipeline.instanceType.regionId}") Long defaultRegionId, + CloudPipelineAPIClient cloudPipelineAPIClient, MessageHelper messageHelper) { this.defaultHddSize = hddSize; + this.defaultToolId = defaultToolId; + this.defaultRegionId = defaultRegionId; + this.cloudPipelineAPIClient = cloudPipelineAPIClient; + this.messageHelper = messageHelper; + } public PipelineStart mapToPipelineStart(TesTask tesTask) { @@ -50,7 +59,7 @@ public PipelineStart mapToPipelineStart(TesTask tesTask) { pipelineStart.setDockerImage(tesExecutor.getImage()); pipelineStart.setExecutionEnvironment(ExecutionEnvironment.CLOUD_PLATFORM); pipelineStart.setHddSize(defaultHddSize); - pipelineStart.setInstanceType(defaultInstanceType); + pipelineStart.setInstanceType(getProperInstanceType(tesTask)); pipelineStart.setIsSpot(tesTask.getResources().getPreemptible()); pipelineStart.setForce(false); pipelineStart.setNonPause(true); @@ -69,4 +78,42 @@ private TesExecutor getExecutorFromTesExecutorsList(List tesExecuto "executors")); return tesExecutors.get(FIRST); } + + private String getProperInstanceType(TesTask tesTask) { + Double ramGb = tesTask.getResources().getRamGb(); + Long cpuCores = tesTask.getResources().getCpuCores(); + //TODO Need clear mappping to loadAllowedInstanceAndPriceTypes() method + //TODO toolId, regionId and spot + Long toolId = 4L; + Long regionId = Long.valueOf(String.join("", tesTask.getResources().getZones())); + HashMap initialTypesMap = + loadAllowedInstanceAndPriceTypes(toolId, regionId, + tesTask.getResources().getPreemptible()); + return evaluateInstanceTypeFromMapOfTypes(ramGb, cpuCores, initialTypesMap); + } + + public HashMap loadAllowedInstanceAndPriceTypes(Long toolId, + Long regionId, Boolean spot) { + HashMap allowedInstanceTypesMap = new HashMap<>(); + AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = + cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(toolId, regionId, spot); + allowedInstanceAndPriceTypes.getAllowedInstanceTypes().forEach(instanceType -> + allowedInstanceTypesMap.put(instanceType.getName(), Collections.singletonMap(instanceType.getMemory(), + instanceType.getVCPU()))); + return allowedInstanceTypesMap; + } + + private String evaluateInstanceTypeFromMapOfTypes(Double ramGb, Long cpuCores, + HashMap initialTypesMap) { + Map weightedTypesMap = new HashMap<>(); + initialTypesMap.forEach((instanceName, instanceParams) -> + weightedTypesMap.put(instanceName, calculateInstanceWeight(instanceParams, ramGb, cpuCores))); + return weightedTypesMap.entrySet() + .stream().min(Comparator.comparing(Map.Entry::getValue)).get().getKey(); + } + + private Double calculateInstanceWeight(Map instanceParamsMap, Double ramGb, Long cpuCores) { + Map.Entry doubleLongEntry = instanceParamsMap.entrySet().iterator().next(); + return Math.abs((doubleLongEntry.getKey() / ramGb + doubleLongEntry.getValue() / cpuCores) / 2 - 1); + } } diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index 6027873e50..2adfda66d6 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -1,5 +1,6 @@ server.port=8080 cloud.pipeline.host=${API} cloud.pipeline.token=${API_TOKEN} -cloud.pipeline.instanceType=m5.large +cloud.pipeline.instanceType.toolId=2 +cloud.pipeline.instanceType.regionId=2 cloud.pipeline.hddSize=20 \ No newline at end of file diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java index 2c64023c78..2be56b3f77 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java @@ -1,6 +1,7 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.tesadapter.common.MessageHelper; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -9,11 +10,15 @@ class TesTaskServiceImplTest { private CloudPipelineAPIClient cloudPipelineAPIClient; private TesTaskServiceImpl tesTaskService; + private TaskMapper taskMapper; + private MessageHelper messageHelper; @BeforeEach public void setUp() { cloudPipelineAPIClient = Mockito.mock(CloudPipelineAPIClient.class); - tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient); + taskMapper = Mockito.mock(TaskMapper.class); + messageHelper = Mockito.mock(MessageHelper.class); + tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient, taskMapper, messageHelper); } @Test From c4caf7c75157dcb018b0c4ab14b17b1782b0ce5b Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 5 Sep 2019 16:56:56 +0300 Subject: [PATCH 063/194] updated TaskMapper, refactored, optimized ramGb + vCPU mapper implementation --- .../service/CloudPipelineAPIClient.java | 12 +++ .../tesadapter/service/TaskMapper.java | 90 +++++++++++-------- 2 files changed, 63 insertions(+), 39 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index ead7aac3f7..0ed50e2df8 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -4,12 +4,16 @@ import com.epam.pipeline.client.pipeline.CloudPipelineApiBuilder; import com.epam.pipeline.entity.cluster.AllowedInstanceAndPriceTypes; import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.entity.pipeline.Tool; import com.epam.pipeline.entity.pipeline.run.PipelineStart; +import com.epam.pipeline.entity.region.AbstractCloudRegion; import com.epam.pipeline.utils.QueryUtils; import com.epam.pipeline.vo.RunStatusVO; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import java.util.List; + @Service public class CloudPipelineAPIClient { private CloudPipelineAPI cloudPipelineAPI; @@ -37,4 +41,12 @@ public AllowedInstanceAndPriceTypes loadAllowedInstanceAndPriceTypes(final Long final Boolean spot) { return QueryUtils.execute(cloudPipelineAPI.loadAllowedInstanceAndPriceTypes(toolId, regionId, spot)); } + + public Tool loadTool(String image) { + return QueryUtils.execute(cloudPipelineAPI.loadTool(null, image)); + } + + public List loadAllRegions(){ + return QueryUtils.execute(cloudPipelineAPI.loadAllRegions()); + } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index bb5259c62c..dd310a8547 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -1,8 +1,10 @@ package com.epam.pipeline.tesadapter.service; import com.epam.pipeline.entity.cluster.AllowedInstanceAndPriceTypes; +import com.epam.pipeline.entity.cluster.InstanceType; import com.epam.pipeline.entity.configuration.ExecutionEnvironment; import com.epam.pipeline.entity.configuration.PipeConfValueVO; +import com.epam.pipeline.entity.pipeline.Tool; import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.tesadapter.common.MessageConstants; import com.epam.pipeline.tesadapter.common.MessageHelper; @@ -16,18 +18,16 @@ import org.springframework.stereotype.Service; import org.springframework.util.Assert; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Slf4j @Service public class TaskMapper { private final Integer defaultHddSize; - private final Long defaultToolId; - private final Long defaultRegionId; private MessageHelper messageHelper; private final CloudPipelineAPIClient cloudPipelineAPIClient; @@ -35,31 +35,36 @@ public class TaskMapper { private static final String INPUT_TYPE = "input"; private static final String OUTPUT_TYPE = "output"; private static final String DEFAULT_TYPE = "string"; + private static final String IMAGE = "image"; + private static final String EXECUTORS = "executors"; + private static final String ZONES = "zones"; private static final Integer FIRST = 0; + private static final Integer ONLY_ONE = 1; + private static final Double GiB_TO_GiB = 1.0; + private static final Double MiB_TO_GiB = 0.0009765625; + @Autowired public TaskMapper(@Value("${cloud.pipeline.hddSize}") Integer hddSize, - @Value("${cloud.pipeline.instanceType.toolId}") Long defaultToolId, - @Value("${cloud.pipeline.instanceType.regionId}") Long defaultRegionId, CloudPipelineAPIClient cloudPipelineAPIClient, MessageHelper messageHelper) { this.defaultHddSize = hddSize; - this.defaultToolId = defaultToolId; - this.defaultRegionId = defaultRegionId; this.cloudPipelineAPIClient = cloudPipelineAPIClient; this.messageHelper = messageHelper; - } public PipelineStart mapToPipelineStart(TesTask tesTask) { PipelineStart pipelineStart = new PipelineStart(); Map params = new HashMap<>(); TesExecutor tesExecutor = getExecutorFromTesExecutorsList(tesTask.getExecutors()); + Assert.notNull(tesExecutor.getImage(), messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, IMAGE)); + Tool pipelineTool = loadToolByTesImage(tesExecutor.getImage()); + pipelineStart.setInstanceType(getProperInstanceType(tesTask, pipelineTool)); pipelineStart.setCmdTemplate(String.join(SEPARATOR, tesExecutor.getCommand())); pipelineStart.setDockerImage(tesExecutor.getImage()); pipelineStart.setExecutionEnvironment(ExecutionEnvironment.CLOUD_PLATFORM); pipelineStart.setHddSize(defaultHddSize); - pipelineStart.setInstanceType(getProperInstanceType(tesTask)); pipelineStart.setIsSpot(tesTask.getResources().getPreemptible()); pipelineStart.setForce(false); pipelineStart.setNonPause(true); @@ -74,46 +79,53 @@ public PipelineStart mapToPipelineStart(TesTask tesTask) { } private TesExecutor getExecutorFromTesExecutorsList(List tesExecutors) { - Assert.notEmpty(tesExecutors, messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, - "executors")); + Assert.isTrue(tesExecutors.size() == ONLY_ONE, messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, EXECUTORS, tesExecutors)); return tesExecutors.get(FIRST); } - private String getProperInstanceType(TesTask tesTask) { + private String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { Double ramGb = tesTask.getResources().getRamGb(); Long cpuCores = tesTask.getResources().getCpuCores(); - //TODO Need clear mappping to loadAllowedInstanceAndPriceTypes() method - //TODO toolId, regionId and spot - Long toolId = 4L; - Long regionId = Long.valueOf(String.join("", tesTask.getResources().getZones())); - HashMap initialTypesMap = - loadAllowedInstanceAndPriceTypes(toolId, regionId, - tesTask.getResources().getPreemptible()); - return evaluateInstanceTypeFromMapOfTypes(ramGb, cpuCores, initialTypesMap); + Long toolId = pipelineTool.getId(); + Long regionId = getProperRegionIdInCloudRegionsByTesZone(tesTask.getResources().getZones()); + Boolean spot = tesTask.getResources().getPreemptible(); + + AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = cloudPipelineAPIClient + .loadAllowedInstanceAndPriceTypes(toolId, regionId, spot); + + return evaluateMostProperInstanceType(allowedInstanceAndPriceTypes, ramGb, cpuCores); + } + + private Tool loadToolByTesImage(String image) { + return cloudPipelineAPIClient.loadTool(image); + } + + private Long getProperRegionIdInCloudRegionsByTesZone(List zones) { + Assert.isTrue(zones.size() == ONLY_ONE, messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, ZONES, zones)); + return cloudPipelineAPIClient.loadAllRegions().stream().filter( + region -> region.getName().equalsIgnoreCase(zones.get(FIRST))) + .collect(Collectors.toList()).get(FIRST).getId(); } - public HashMap loadAllowedInstanceAndPriceTypes(Long toolId, - Long regionId, Boolean spot) { - HashMap allowedInstanceTypesMap = new HashMap<>(); - AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = - cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(toolId, regionId, spot); - allowedInstanceAndPriceTypes.getAllowedInstanceTypes().forEach(instanceType -> - allowedInstanceTypesMap.put(instanceType.getName(), Collections.singletonMap(instanceType.getMemory(), - instanceType.getVCPU()))); - return allowedInstanceTypesMap; + public String evaluateMostProperInstanceType(AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes, + Double ramGb, Long cpuCores) { + return allowedInstanceAndPriceTypes.getAllowedInstanceTypes().stream() + .collect(Collectors.toMap(InstanceType::getName, instancaType -> + calculateInstanceCoef(instancaType, ramGb, cpuCores))).entrySet().stream() + .min(Comparator.comparing(Map.Entry::getValue)).orElseThrow(NullPointerException::new).getKey(); } - private String evaluateInstanceTypeFromMapOfTypes(Double ramGb, Long cpuCores, - HashMap initialTypesMap) { - Map weightedTypesMap = new HashMap<>(); - initialTypesMap.forEach((instanceName, instanceParams) -> - weightedTypesMap.put(instanceName, calculateInstanceWeight(instanceParams, ramGb, cpuCores))); - return weightedTypesMap.entrySet() - .stream().min(Comparator.comparing(Map.Entry::getValue)).get().getKey(); + private Double calculateInstanceCoef(InstanceType instanceType, Double ramGb, Long cpuCores) { + return Math.abs((instanceType.getMemory() * parseInstanceMemoryUnit(instanceType.getMemoryUnit()) + / ramGb + instanceType.getVCPU() / cpuCores) / 2 - 1); } - private Double calculateInstanceWeight(Map instanceParamsMap, Double ramGb, Long cpuCores) { - Map.Entry doubleLongEntry = instanceParamsMap.entrySet().iterator().next(); - return Math.abs((doubleLongEntry.getKey() / ramGb + doubleLongEntry.getValue() / cpuCores) / 2 - 1); + private Double parseInstanceMemoryUnit(String memoryUnit) { + if (memoryUnit != null && memoryUnit.equalsIgnoreCase("MiB")) { + return MiB_TO_GiB; + } + return GiB_TO_GiB; } } From eb14e2fe276e760a018e2e1fdddd6c51d55e2aa1 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Thu, 5 Sep 2019 18:34:50 +0300 Subject: [PATCH 064/194] refactored method parseRunId and some little things --- .../controller/TesAdapterController.java | 21 ++---------- .../tesadapter/service/TaskMapper.java | 34 +++++++------------ .../tesadapter/service/TesTaskService.java | 3 +- .../service/TesTaskServiceImpl.java | 22 ++++-------- .../service/TesTaskServiceImplTest.java | 7 +++- 5 files changed, 28 insertions(+), 59 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index 87c3ab52b0..1d0f74dbea 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -1,7 +1,5 @@ package com.epam.pipeline.tesadapter.controller; -import com.epam.pipeline.tesadapter.common.MessageConstants; -import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.entity.TaskView; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; @@ -12,7 +10,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.util.Assert; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; @@ -24,11 +21,9 @@ @RestController public class TesAdapterController { private TesTaskService tesTaskService; - private MessageHelper messageHelper; @Autowired - public TesAdapterController(TesTaskService tesTaskService, MessageHelper messageHelper) { - this.messageHelper = messageHelper; + public TesAdapterController(TesTaskService tesTaskService) { this.tesTaskService = tesTaskService; } @@ -55,23 +50,11 @@ ResponseEntity submitTesTask(@RequestBody TesTask body) { @GetMapping("/v1/tasks/{id}") ResponseEntity getTesTask(@PathVariable String id, @RequestParam(required = false, defaultValue = "MINIMAL") TaskView view) { - return ResponseEntity.ok().body(tesTaskService.getTesTask(parseStringIdToLongId(id))); - + return ResponseEntity.ok().body(tesTaskService.getTesTask(id)); } @PostMapping("/v1/tasks/{id}:cancel") ResponseEntity cancelTesTask(@PathVariable String id) { return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.cancelTesTask(id)); } - - private Long parseStringIdToLongId(String id){ - Assert.hasText(id, messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_NON_SCALAR_TYPE)); - try { - return Long.parseLong(id); - } catch (NumberFormatException e) { - log.error(messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, - "ID", id)); - throw new IllegalArgumentException(e); - } - } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index bdd2b8ba55..8df11a419a 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -31,12 +31,9 @@ @Service public class TaskMapper { private final String defaultInstanceType; - private final Integer defaultHddSize; - @Autowired private MessageHelper messageHelper; - private static final String SEPARATOR = " "; private static final String INPUT_TYPE = "input"; private static final String OUTPUT_TYPE = "output"; @@ -91,44 +88,39 @@ public TesTask mapToTesTask(PipelineRun run) { return tesTask; } - private TesState createTesState(TaskStatus status){ - TesState tesStatus = TesState.UNKNOWN; - switch (status){ + private TesState createTesState(TaskStatus status) { + switch (status) { case RUNNING: - tesStatus = TesState.RUNNING; - break; + return TesState.RUNNING; case PAUSED: - tesStatus = TesState.PAUSED; - break; + return TesState.PAUSED; case SUCCESS: - tesStatus = TesState.COMPLETE; - break; + return TesState.COMPLETE; case FAILURE: - tesStatus = TesState.EXECUTOR_ERROR; - break; + return TesState.EXECUTOR_ERROR; case STOPPED: - tesStatus = TesState.CANCELED; - break; + return TesState.CANCELED; + default: + return TesState.UNKNOWN; } - return tesStatus; } - private List createTesInput(List parameters){ + private List createTesInput(List parameters) { final TesInput tesInput = new TesInput(); parameters.stream() .filter(pipelineRunParameter -> pipelineRunParameter.getType().contains(INPUT_TYPE)) - .peek(pipelineRunParameter -> { + .forEach(pipelineRunParameter -> { tesInput.setName(pipelineRunParameter.getName()); tesInput.setUrl(pipelineRunParameter.getValue()); }); return ListUtils.emptyIfNull(Arrays.asList(tesInput)); } - private List createTesOutput(List parameters){ + private List createTesOutput(List parameters) { final TesOutput tesOutput = new TesOutput(); parameters.stream() .filter(pipelineRunParameter -> pipelineRunParameter.getType().contains(OUTPUT_TYPE)) - .peek(pipelineRunParameter -> { + .forEach(pipelineRunParameter -> { tesOutput.setName(pipelineRunParameter.getName()); tesOutput.setUrl(pipelineRunParameter.getValue()); }); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index 9420a9b15b..e48b7c4048 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -6,7 +6,6 @@ import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; public interface TesTaskService { - TesCreateTaskResponse submitTesTask(TesTask body); TesListTasksResponse listTesTask(); @@ -15,5 +14,5 @@ public interface TesTaskService { TesCancelTaskResponse cancelTesTask(String id); - TesTask getTesTask(Long id); + TesTask getTesTask(String id); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 9962e5248f..42066cfcf6 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -1,6 +1,5 @@ package com.epam.pipeline.tesadapter.service; - import com.epam.pipeline.entity.pipeline.PipelineRun; import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.tesadapter.common.MessageConstants; @@ -11,6 +10,7 @@ import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.vo.RunStatusVO; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.Assert; @@ -19,11 +19,8 @@ @Service public class TesTaskServiceImpl implements TesTaskService { private final CloudPipelineAPIClient cloudPipelineAPIClient; - private final TaskMapper taskMapper; - private final MessageHelper messageHelper; - private final static String ID = "id"; @Autowired @@ -63,20 +60,13 @@ public TesCancelTaskResponse cancelTesTask(String id) { } @Override - public TesTask getTesTask(Long id) { - return taskMapper.mapToTesTask(cloudPipelineAPIClient.loadPipelineRun(id)); - + public TesTask getTesTask(String id) { + return taskMapper.mapToTesTask(cloudPipelineAPIClient.loadPipelineRun(parseRunId(id))); } private Long parseRunId(String id) { - Assert.hasText(id, messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_NON_SCALAR_TYPE, - ID, id)); - try { - return Long.parseLong(id); - } catch (NumberFormatException e) { - log.error(messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, - ID, id)); - throw new IllegalArgumentException(e); - } + Assert.state(StringUtils.isNumeric(id), + messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, "ID", id)); + return Long.parseLong(id); } } diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java index 2c64023c78..2be56b3f77 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java @@ -1,6 +1,7 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.tesadapter.common.MessageHelper; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -9,11 +10,15 @@ class TesTaskServiceImplTest { private CloudPipelineAPIClient cloudPipelineAPIClient; private TesTaskServiceImpl tesTaskService; + private TaskMapper taskMapper; + private MessageHelper messageHelper; @BeforeEach public void setUp() { cloudPipelineAPIClient = Mockito.mock(CloudPipelineAPIClient.class); - tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient); + taskMapper = Mockito.mock(TaskMapper.class); + messageHelper = Mockito.mock(MessageHelper.class); + tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient, taskMapper, messageHelper); } @Test From 1aa242c00bf70399daf4fa9827140bf74a13301f Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 5 Sep 2019 19:02:58 +0300 Subject: [PATCH 065/194] couple of clean-code in TaskMapper, fixed bug in AllowedInstanceAndPriceTypes class in common, added "cluster." key to @JsonProperty --- .../entity/cluster/AllowedInstanceAndPriceTypes.java | 6 +++--- .../epam/pipeline/tesadapter/service/TaskMapper.java | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/entity/cluster/AllowedInstanceAndPriceTypes.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/entity/cluster/AllowedInstanceAndPriceTypes.java index 93810278d5..0be757a102 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/entity/cluster/AllowedInstanceAndPriceTypes.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/entity/cluster/AllowedInstanceAndPriceTypes.java @@ -23,12 +23,12 @@ @Value public class AllowedInstanceAndPriceTypes { - @JsonProperty("allowed.instance.types") + @JsonProperty("cluster.allowed.instance.types") private final List allowedInstanceTypes; - @JsonProperty("allowed.instance.docker.types") + @JsonProperty("cluster.allowed.instance.docker.types") private final List allowedInstanceDockerTypes; - @JsonProperty("allowed.price.types") + @JsonProperty("cluster.allowed.price.types") private final List allowedPriceTypes; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index dd310a8547..18ca791ce7 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import java.util.stream.Collectors; @Slf4j @@ -38,6 +39,7 @@ public class TaskMapper { private static final String IMAGE = "image"; private static final String EXECUTORS = "executors"; private static final String ZONES = "zones"; + private static final String MiB = "MiB"; private static final Integer FIRST = 0; private static final Integer ONLY_ONE = 1; private static final Double GiB_TO_GiB = 1.0; @@ -93,7 +95,8 @@ private String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = cloudPipelineAPIClient .loadAllowedInstanceAndPriceTypes(toolId, regionId, spot); - + Assert.isTrue(allowedInstanceAndPriceTypes.getAllowedInstanceTypes() != null, messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, allowedInstanceAndPriceTypes)); return evaluateMostProperInstanceType(allowedInstanceAndPriceTypes, ramGb, cpuCores); } @@ -109,8 +112,8 @@ private Long getProperRegionIdInCloudRegionsByTesZone(List zones) { .collect(Collectors.toList()).get(FIRST).getId(); } - public String evaluateMostProperInstanceType(AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes, - Double ramGb, Long cpuCores) { + private String evaluateMostProperInstanceType(AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes, + Double ramGb, Long cpuCores) { return allowedInstanceAndPriceTypes.getAllowedInstanceTypes().stream() .collect(Collectors.toMap(InstanceType::getName, instancaType -> calculateInstanceCoef(instancaType, ramGb, cpuCores))).entrySet().stream() @@ -123,7 +126,7 @@ private Double calculateInstanceCoef(InstanceType instanceType, Double ramGb, Lo } private Double parseInstanceMemoryUnit(String memoryUnit) { - if (memoryUnit != null && memoryUnit.equalsIgnoreCase("MiB")) { + if (memoryUnit != null && memoryUnit.equalsIgnoreCase(MiB)) { return MiB_TO_GiB; } return GiB_TO_GiB; From 9d905af33bad4ecce4f19b2177508ebaf1279625 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 5 Sep 2019 19:04:29 +0300 Subject: [PATCH 066/194] fixed pmdMain, removed unnecessary imports. --- .../java/com/epam/pipeline/tesadapter/service/TaskMapper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 18ca791ce7..217e48633d 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -22,7 +22,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.TreeMap; import java.util.stream.Collectors; @Slf4j From 79114795fb0683c79c48f9c5b2258fe7c702f5ef Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Thu, 5 Sep 2019 19:59:23 +0300 Subject: [PATCH 067/194] did merge with tes-support and solved conflicts --- .../pipeline/tesadapter/service/TesTaskServiceImpl.java | 5 +---- .../tesadapter/service/TesTaskServiceImplTest.java | 7 ++++++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index f54989a5b5..b13d6c3389 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -1,6 +1,6 @@ package com.epam.pipeline.tesadapter.service; -import com.epam.pipeline.entity.datastorage.AbstractDataStorage; +import com.epam.pipeline.entity.pipeline.PipelineRun; import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.tesadapter.common.MessageConstants; import com.epam.pipeline.tesadapter.common.MessageHelper; @@ -28,11 +28,8 @@ public class TesTaskServiceImpl implements TesTaskService { @Value("${cloud.pipeline.doc}") private String doc; private final CloudPipelineAPIClient cloudPipelineAPIClient; - private final TaskMapper taskMapper; - private final MessageHelper messageHelper; - private final static String ID = "id"; @Autowired diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java index 2c64023c78..656106469b 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java @@ -1,6 +1,7 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.tesadapter.common.MessageHelper; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -9,11 +10,15 @@ class TesTaskServiceImplTest { private CloudPipelineAPIClient cloudPipelineAPIClient; private TesTaskServiceImpl tesTaskService; + private MessageHelper messageHelper; + private TaskMapper taskMapper; @BeforeEach public void setUp() { cloudPipelineAPIClient = Mockito.mock(CloudPipelineAPIClient.class); - tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient); + messageHelper = Mockito.mock(MessageHelper.class); + taskMapper = Mockito.mock(TaskMapper.class); + tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient, taskMapper, messageHelper); } @Test From ea29bcff842b08f32bb45caf61a92e39d6b991e3 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 6 Sep 2019 14:48:06 +0300 Subject: [PATCH 068/194] couple of clean-code in TaskMapper, added mapping disk_Gb(TES) to hddSize(CP), added defaultValues to ramGb and cpuCores params. added java-docs to calculateInstanceCoef() method. Added enum PipelineDiskMemoryTypes with correspond types of memory in Cloud-Pipeline. --- .../entity/PipelineDiskMemoryTypes.java | 25 ++++++++ .../tesadapter/service/TaskMapper.java | 60 ++++++++++++++----- .../src/main/resources/application.properties | 6 +- 3 files changed, 72 insertions(+), 19 deletions(-) create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/PipelineDiskMemoryTypes.java diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/PipelineDiskMemoryTypes.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/PipelineDiskMemoryTypes.java new file mode 100644 index 0000000000..ccf7f25e5b --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/PipelineDiskMemoryTypes.java @@ -0,0 +1,25 @@ +package com.epam.pipeline.tesadapter.entity; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.ToString; + +@AllArgsConstructor +@Getter +@ToString +public enum PipelineDiskMemoryTypes { + KIB("KiB"), + GIB("GiB"), + MIB("MiB"), + TIB("TiB"), + PIB("PiB"), + EIB("EiB"), + KIB_TO_GIB("0.00000095367432"), + MIB_TO_GIB("0.0009765625"), + GIB_TO_GIB("1"), + TIB_TO_GIB("1024"), + PIB_TO_GIB("1048576"), + EIB_TO_GIB("1073741824"); + + private String value; +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 217e48633d..0b4eebef2d 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -8,6 +8,7 @@ import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.tesadapter.common.MessageConstants; import com.epam.pipeline.tesadapter.common.MessageHelper; +import com.epam.pipeline.tesadapter.entity.PipelineDiskMemoryTypes; import com.epam.pipeline.tesadapter.entity.TesExecutor; import com.epam.pipeline.tesadapter.entity.TesTask; import lombok.extern.slf4j.Slf4j; @@ -28,6 +29,8 @@ @Service public class TaskMapper { private final Integer defaultHddSize; + private final Double defaultRamGb; + private final Long defaultCpuCore; private MessageHelper messageHelper; private final CloudPipelineAPIClient cloudPipelineAPIClient; @@ -38,17 +41,18 @@ public class TaskMapper { private static final String IMAGE = "image"; private static final String EXECUTORS = "executors"; private static final String ZONES = "zones"; - private static final String MiB = "MiB"; private static final Integer FIRST = 0; private static final Integer ONLY_ONE = 1; - private static final Double GiB_TO_GiB = 1.0; - private static final Double MiB_TO_GiB = 0.0009765625; @Autowired public TaskMapper(@Value("${cloud.pipeline.hddSize}") Integer hddSize, + @Value("${cloud.pipeline.ramGb}") Double defaultRamGb, + @Value("${cloud.pipeline.cpuCore}") Long defaultCpuCore, CloudPipelineAPIClient cloudPipelineAPIClient, MessageHelper messageHelper) { this.defaultHddSize = hddSize; + this.defaultRamGb = defaultRamGb; + this.defaultCpuCore = defaultCpuCore; this.cloudPipelineAPIClient = cloudPipelineAPIClient; this.messageHelper = messageHelper; } @@ -65,7 +69,8 @@ public PipelineStart mapToPipelineStart(TesTask tesTask) { pipelineStart.setCmdTemplate(String.join(SEPARATOR, tesExecutor.getCommand())); pipelineStart.setDockerImage(tesExecutor.getImage()); pipelineStart.setExecutionEnvironment(ExecutionEnvironment.CLOUD_PLATFORM); - pipelineStart.setHddSize(defaultHddSize); + pipelineStart.setHddSize(tesTask.getResources().getDiskGb() != null ? + tesTask.getResources().getDiskGb().intValue() : defaultHddSize); pipelineStart.setIsSpot(tesTask.getResources().getPreemptible()); pipelineStart.setForce(false); pipelineStart.setNonPause(true); @@ -86,15 +91,17 @@ private TesExecutor getExecutorFromTesExecutorsList(List tesExecuto } private String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { - Double ramGb = tesTask.getResources().getRamGb(); - Long cpuCores = tesTask.getResources().getCpuCores(); + Double ramGb = + tesTask.getResources().getRamGb() != null ? tesTask.getResources().getRamGb() : defaultRamGb; + Long cpuCores = + tesTask.getResources().getCpuCores() != null ? tesTask.getResources().getCpuCores() : defaultCpuCore; Long toolId = pipelineTool.getId(); Long regionId = getProperRegionIdInCloudRegionsByTesZone(tesTask.getResources().getZones()); Boolean spot = tesTask.getResources().getPreemptible(); AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = cloudPipelineAPIClient .loadAllowedInstanceAndPriceTypes(toolId, regionId, spot); - Assert.isTrue(allowedInstanceAndPriceTypes.getAllowedInstanceTypes() != null, messageHelper.getMessage( + Assert.notEmpty(allowedInstanceAndPriceTypes.getAllowedInstanceTypes(), messageHelper.getMessage( MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, allowedInstanceAndPriceTypes)); return evaluateMostProperInstanceType(allowedInstanceAndPriceTypes, ramGb, cpuCores); } @@ -114,20 +121,41 @@ private Long getProperRegionIdInCloudRegionsByTesZone(List zones) { private String evaluateMostProperInstanceType(AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes, Double ramGb, Long cpuCores) { return allowedInstanceAndPriceTypes.getAllowedInstanceTypes().stream() - .collect(Collectors.toMap(InstanceType::getName, instancaType -> - calculateInstanceCoef(instancaType, ramGb, cpuCores))).entrySet().stream() - .min(Comparator.comparing(Map.Entry::getValue)).orElseThrow(NullPointerException::new).getKey(); + .min(Comparator.comparing(i -> calculateInstanceCoef(i, ramGb, cpuCores))) + .orElseThrow(IllegalArgumentException::new) + .getName(); } + /** + * Calculates the effective coefficient for {@code instanceType}.The result + * depends on the level of difference between the used values {@code memory} and + * {@code vCPU} in {@code instanceType} and the entered values {@code ramGb} and + * {@code cpuCores}, respectively. From greater difference, comes greater coefficient. + * + * @param instanceType InstanceType - a set of using parameters + * @param ramGb double - entered RAM (Gb) as resource parameter + * @param cpuCores long - entered CPU (Cores) as resource parameter + * @return double - correspond coefficient for {@code instanceType} + */ private Double calculateInstanceCoef(InstanceType instanceType, Double ramGb, Long cpuCores) { - return Math.abs((instanceType.getMemory() * parseInstanceMemoryUnit(instanceType.getMemoryUnit()) - / ramGb + instanceType.getVCPU() / cpuCores) / 2 - 1); + return Math.abs((convertMemoryUnitTypeToGiB(instanceType.getMemoryUnit()) * instanceType.getMemory() + / ramGb + (double) instanceType.getVCPU() / (double) cpuCores) / 2 - 1); } - private Double parseInstanceMemoryUnit(String memoryUnit) { - if (memoryUnit != null && memoryUnit.equalsIgnoreCase(MiB)) { - return MiB_TO_GiB; + private Double convertMemoryUnitTypeToGiB(String memoryUnit) { + if (memoryUnit != null) { + if (memoryUnit.equalsIgnoreCase(PipelineDiskMemoryTypes.KIB.getValue())) { + return Double.valueOf(PipelineDiskMemoryTypes.KIB_TO_GIB.getValue()); + } else if (memoryUnit.equalsIgnoreCase(PipelineDiskMemoryTypes.MIB.getValue())) { + return Double.valueOf(PipelineDiskMemoryTypes.MIB_TO_GIB.getValue()); + } else if (memoryUnit.equalsIgnoreCase(PipelineDiskMemoryTypes.TIB.getValue())) { + return Double.valueOf(PipelineDiskMemoryTypes.TIB_TO_GIB.getValue()); + } else if (memoryUnit.equalsIgnoreCase(PipelineDiskMemoryTypes.PIB.getValue())) { + return Double.valueOf(PipelineDiskMemoryTypes.PIB_TO_GIB.getValue()); + } else if (memoryUnit.equalsIgnoreCase(PipelineDiskMemoryTypes.EIB.getValue())) { + return Double.valueOf(PipelineDiskMemoryTypes.EIB_TO_GIB.getValue()); + } } - return GiB_TO_GiB; + return Double.valueOf(PipelineDiskMemoryTypes.GIB_TO_GIB.getValue()); } } diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index 2adfda66d6..fe90a39080 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -1,6 +1,6 @@ server.port=8080 cloud.pipeline.host=${API} cloud.pipeline.token=${API_TOKEN} -cloud.pipeline.instanceType.toolId=2 -cloud.pipeline.instanceType.regionId=2 -cloud.pipeline.hddSize=20 \ No newline at end of file +cloud.pipeline.ramGb=16 +cloud.pipeline.cpuCore=2 +cloud.pipeline.hddSize=50 \ No newline at end of file From 21446b52ed23de5b35fa47c0f25139fe6a1b9a82 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 6 Sep 2019 15:34:29 +0300 Subject: [PATCH 069/194] clean-code in TesTaskServiceImplTest, replace global-to-local fields taskMapper and messageHelper --- .../pipeline/tesadapter/service/TesTaskServiceImplTest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java index 2be56b3f77..cc8c5633f3 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java @@ -10,14 +10,12 @@ class TesTaskServiceImplTest { private CloudPipelineAPIClient cloudPipelineAPIClient; private TesTaskServiceImpl tesTaskService; - private TaskMapper taskMapper; - private MessageHelper messageHelper; @BeforeEach public void setUp() { cloudPipelineAPIClient = Mockito.mock(CloudPipelineAPIClient.class); - taskMapper = Mockito.mock(TaskMapper.class); - messageHelper = Mockito.mock(MessageHelper.class); + TaskMapper taskMapper = Mockito.mock(TaskMapper.class); + MessageHelper messageHelper = Mockito.mock(MessageHelper.class); tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient, taskMapper, messageHelper); } From 3d128754eeb4cae416057b723ed8eae2842439a1 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 6 Sep 2019 16:59:51 +0300 Subject: [PATCH 070/194] refactoring in TaskMapper. added defaultValue for "preemptible" parameter< added empty TaskMapperTest class. refactored PipelineDiskMemoryTypes, replaced constants for converting. Implemented Optional.ofNullable() to avoid "null" in TesResources object. --- .../entity/PipelineDiskMemoryTypes.java | 8 +------- .../tesadapter/service/TaskMapper.java | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/PipelineDiskMemoryTypes.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/PipelineDiskMemoryTypes.java index ccf7f25e5b..03f3288c1f 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/PipelineDiskMemoryTypes.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/PipelineDiskMemoryTypes.java @@ -13,13 +13,7 @@ public enum PipelineDiskMemoryTypes { MIB("MiB"), TIB("TiB"), PIB("PiB"), - EIB("EiB"), - KIB_TO_GIB("0.00000095367432"), - MIB_TO_GIB("0.0009765625"), - GIB_TO_GIB("1"), - TIB_TO_GIB("1024"), - PIB_TO_GIB("1048576"), - EIB_TO_GIB("1073741824"); + EIB("EiB"); private String value; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 0b4eebef2d..524eb3f428 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -43,6 +43,12 @@ public class TaskMapper { private static final String ZONES = "zones"; private static final Integer FIRST = 0; private static final Integer ONLY_ONE = 1; + private static final Double KIB_TO_GIB = 0.00000095367432; + private static final Double MIB_TO_GIB = 0.0009765625; + private static final Double GIB_TO_GIB = 1.0; + private static final Double TIB_TO_GIB = 1024.0; + private static final Double PIB_TO_GIB = 1048576.0; + private static final Double EIB_TO_GIB = 1073741824.0; @Autowired @@ -145,17 +151,17 @@ private Double calculateInstanceCoef(InstanceType instanceType, Double ramGb, Lo private Double convertMemoryUnitTypeToGiB(String memoryUnit) { if (memoryUnit != null) { if (memoryUnit.equalsIgnoreCase(PipelineDiskMemoryTypes.KIB.getValue())) { - return Double.valueOf(PipelineDiskMemoryTypes.KIB_TO_GIB.getValue()); + return KIB_TO_GIB; } else if (memoryUnit.equalsIgnoreCase(PipelineDiskMemoryTypes.MIB.getValue())) { - return Double.valueOf(PipelineDiskMemoryTypes.MIB_TO_GIB.getValue()); + return MIB_TO_GIB; } else if (memoryUnit.equalsIgnoreCase(PipelineDiskMemoryTypes.TIB.getValue())) { - return Double.valueOf(PipelineDiskMemoryTypes.TIB_TO_GIB.getValue()); + return TIB_TO_GIB; } else if (memoryUnit.equalsIgnoreCase(PipelineDiskMemoryTypes.PIB.getValue())) { - return Double.valueOf(PipelineDiskMemoryTypes.PIB_TO_GIB.getValue()); + return PIB_TO_GIB; } else if (memoryUnit.equalsIgnoreCase(PipelineDiskMemoryTypes.EIB.getValue())) { - return Double.valueOf(PipelineDiskMemoryTypes.EIB_TO_GIB.getValue()); + return EIB_TO_GIB; } } - return Double.valueOf(PipelineDiskMemoryTypes.GIB_TO_GIB.getValue()); + return GIB_TO_GIB; } } From 43fa8b8d6d4f85ec097b3f1dea6f6c0cd3909588 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 6 Sep 2019 17:01:36 +0300 Subject: [PATCH 071/194] refactoring in TaskMapper. added defaultValue for "preemptible" parameter< added empty TaskMapperTest class --- .../tesadapter/service/TaskMapper.java | 20 ++++++++++++++----- .../src/main/resources/application.properties | 1 + .../tesadapter/service/TaskMapperTest.java | 12 +++++++++++ 3 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 524eb3f428..3c71c019a1 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -10,6 +10,7 @@ import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.entity.PipelineDiskMemoryTypes; import com.epam.pipeline.tesadapter.entity.TesExecutor; +import com.epam.pipeline.tesadapter.entity.TesResources; import com.epam.pipeline.tesadapter.entity.TesTask; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.ListUtils; @@ -23,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; @Slf4j @@ -31,6 +33,7 @@ public class TaskMapper { private final Integer defaultHddSize; private final Double defaultRamGb; private final Long defaultCpuCore; + private final Boolean defaultPreemptible; private MessageHelper messageHelper; private final CloudPipelineAPIClient cloudPipelineAPIClient; @@ -55,18 +58,24 @@ public class TaskMapper { public TaskMapper(@Value("${cloud.pipeline.hddSize}") Integer hddSize, @Value("${cloud.pipeline.ramGb}") Double defaultRamGb, @Value("${cloud.pipeline.cpuCore}") Long defaultCpuCore, + @Value("${cloud.pipeline.preemtible}") Boolean defaultPreemptible, CloudPipelineAPIClient cloudPipelineAPIClient, MessageHelper messageHelper) { this.defaultHddSize = hddSize; this.defaultRamGb = defaultRamGb; this.defaultCpuCore = defaultCpuCore; this.cloudPipelineAPIClient = cloudPipelineAPIClient; this.messageHelper = messageHelper; + this.defaultPreemptible = defaultPreemptible; } public PipelineStart mapToPipelineStart(TesTask tesTask) { + Assert.notNull(tesTask, messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, tesTask)); PipelineStart pipelineStart = new PipelineStart(); Map params = new HashMap<>(); TesExecutor tesExecutor = getExecutorFromTesExecutorsList(tesTask.getExecutors()); + + Assert.notNull(tesExecutor.getImage(), messageHelper.getMessage( MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, IMAGE)); Tool pipelineTool = loadToolByTesImage(tesExecutor.getImage()); @@ -75,9 +84,10 @@ public PipelineStart mapToPipelineStart(TesTask tesTask) { pipelineStart.setCmdTemplate(String.join(SEPARATOR, tesExecutor.getCommand())); pipelineStart.setDockerImage(tesExecutor.getImage()); pipelineStart.setExecutionEnvironment(ExecutionEnvironment.CLOUD_PLATFORM); - pipelineStart.setHddSize(tesTask.getResources().getDiskGb() != null ? - tesTask.getResources().getDiskGb().intValue() : defaultHddSize); - pipelineStart.setIsSpot(tesTask.getResources().getPreemptible()); + pipelineStart.setHddSize(Optional.ofNullable(tesTask.getResources()) + .map(tesResources -> tesResources.getDiskGb().intValue()).orElse(defaultHddSize)); + pipelineStart.setIsSpot(Optional.ofNullable(tesTask.getResources()) + .map(TesResources::getPreemptible).orElse(defaultPreemptible)); pipelineStart.setForce(false); pipelineStart.setNonPause(true); ListUtils.emptyIfNull(tesTask.getInputs()).forEach(tesInput -> @@ -98,9 +108,9 @@ private TesExecutor getExecutorFromTesExecutorsList(List tesExecuto private String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { Double ramGb = - tesTask.getResources().getRamGb() != null ? tesTask.getResources().getRamGb() : defaultRamGb; + Optional.ofNullable(tesTask.getResources()).map(TesResources::getRamGb).orElse(defaultRamGb); Long cpuCores = - tesTask.getResources().getCpuCores() != null ? tesTask.getResources().getCpuCores() : defaultCpuCore; + Optional.ofNullable(tesTask.getResources()).map(TesResources::getCpuCores).orElse(defaultCpuCore); Long toolId = pipelineTool.getId(); Long regionId = getProperRegionIdInCloudRegionsByTesZone(tesTask.getResources().getZones()); Boolean spot = tesTask.getResources().getPreemptible(); diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index 05968c5be8..0754ac9455 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -4,6 +4,7 @@ cloud.pipeline.token=${API_TOKEN} cloud.pipeline.ramGb=16 cloud.pipeline.cpuCore=2 cloud.pipeline.hddSize=50 +cloud.pipeline.preemtible=true cloud.pipeline.service.name=CloudPipeline cloud.pipeline.doc=https://epam.github.io/cloud-pipeline/ diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java new file mode 100644 index 0000000000..a14b946ee2 --- /dev/null +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -0,0 +1,12 @@ +package com.epam.pipeline.tesadapter.service; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class TaskMapperTest { + @Test + void mapToPipelineStartTest() { + + } +} \ No newline at end of file From adf6e22524aaebbf58b74aa260da865b9f2a2b67 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Fri, 6 Sep 2019 17:07:00 +0300 Subject: [PATCH 072/194] write used @Builder TesExecutor, TesResources and TesTask --- .../tesadapter/entity/TesExecutor.java | 4 ++- .../tesadapter/entity/TesResources.java | 2 ++ .../pipeline/tesadapter/entity/TesTask.java | 2 ++ .../tesadapter/service/TaskMapper.java | 36 +++++++++---------- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java index 9f90fdbbe9..6197daa842 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java @@ -3,12 +3,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.Builder; import lombok.Data; import java.util.List; import java.util.Map; @Data +@Builder @ApiModel(description = "Executor describes a command to be executed, and its environment.") public class TesExecutor { @ApiModelProperty(value = "Name of the container image, for example: " + @@ -29,7 +31,7 @@ public class TesExecutor { @ApiModelProperty(value = "Path inside the container to a file which will " + "be piped to the executor's stdin. Must be an absolute path.") @JsonProperty("stdin") - private String stdin = null; + private String stdin; @ApiModelProperty(value = "Path inside the container to a file where the " + "executor's stdout will be written to. Must be an absolute path.") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java index 6ec7cbb466..c45fd1f4c5 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.Builder; import lombok.Data; import javax.validation.Valid; @@ -10,6 +11,7 @@ @ApiModel(description = "Resources describes the resources requested by a task.") @Data +@Builder public class TesResources { @ApiModelProperty(value = "Requested number of CPUs") @JsonProperty("cpu_cores") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java index 2d5802be62..926dc2f3bd 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.Builder; import lombok.Data; import java.util.List; @@ -10,6 +11,7 @@ @ApiModel(description = "Task describes an instance of a task.") @Data +@Builder public class TesTask { @ApiModelProperty(value = "Task identifier assigned by the server.") @JsonProperty("id") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 8df11a419a..5451bc6bb7 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -76,16 +76,16 @@ private TesExecutor getExecutorFromTesExecutorsList(List tesExecuto } public TesTask mapToTesTask(PipelineRun run) { - final TesTask tesTask = new TesTask(); - tesTask.setId(String.valueOf(run.getId())); - tesTask.setState(createTesState(run.getStatus())); - tesTask.setName(run.getPodId()); - tesTask.setResources(createTesResources(run)); - tesTask.setExecutors(createListExecutor(run)); - tesTask.setInputs(createTesInput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))); - tesTask.setOutputs(createTesOutput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))); - tesTask.setCreationTime(run.getStartDate().toString()); - return tesTask; + return TesTask.builder() + .id(String.valueOf(run.getId())) + .state(createTesState(run.getStatus())) + .name(run.getPodId()) + .resources(createTesResources(run)) + .executors(createListExecutor(run)) + .inputs(createTesInput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) + .outputs(createTesOutput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) + .creationTime(run.getStartDate().toString()) + .build(); } private TesState createTesState(TaskStatus status) { @@ -128,16 +128,16 @@ private List createTesOutput(List parameters) { } private TesResources createTesResources(PipelineRun run){ - final TesResources tesResources = new TesResources(); - tesResources.setPreemptible(run.getInstance().getSpot()); - tesResources.setDiskGb(new Double(run.getInstance().getNodeDisk())); - return tesResources; + return TesResources.builder() + .preemptible(run.getInstance().getSpot()) + .diskGb(new Double(run.getInstance().getNodeDisk())) + .build(); } private List createListExecutor(PipelineRun run){ - final TesExecutor tesExecutor = new TesExecutor(); - tesExecutor.setCommand(ListUtils.emptyIfNull(Arrays.asList(run.getActualCmd().split(SEPARATOR)))); - tesExecutor.setEnv(run.getEnvVars()); - return ListUtils.emptyIfNull(Arrays.asList(tesExecutor)); + return ListUtils.emptyIfNull(Arrays.asList(TesExecutor.builder() + .command(ListUtils.emptyIfNull(Arrays.asList(run.getActualCmd().split(SEPARATOR)))) + .env(run.getEnvVars()) + .build())); } } From 34aa8ce65f8bcb149c37720d2b6d998864c4a3ad Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Fri, 6 Sep 2019 18:13:23 +0300 Subject: [PATCH 073/194] refactoring UnitTest TesTaskServiceImpl --- .../pipeline/tesadapter/service/TesTaskServiceImplTest.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java index 2be56b3f77..02c53ea2af 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java @@ -10,15 +10,11 @@ class TesTaskServiceImplTest { private CloudPipelineAPIClient cloudPipelineAPIClient; private TesTaskServiceImpl tesTaskService; - private TaskMapper taskMapper; - private MessageHelper messageHelper; @BeforeEach public void setUp() { cloudPipelineAPIClient = Mockito.mock(CloudPipelineAPIClient.class); - taskMapper = Mockito.mock(TaskMapper.class); - messageHelper = Mockito.mock(MessageHelper.class); - tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient, taskMapper, messageHelper); + tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient, Mockito.mock(TaskMapper.class), Mockito.mock(MessageHelper.class)); } @Test From e72e5969d109094841f9cc176b2aeb1708667097 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 6 Sep 2019 18:25:16 +0300 Subject: [PATCH 074/194] refactoring in TaskMapper. Now, whole "resources{}" part stubbed with default values if correspond request sends without resources part in body --- .../epam/pipeline/tesadapter/service/TaskMapper.java | 11 +++++++++-- tes-adapter/src/main/resources/application.properties | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 3c71c019a1..2060d14c87 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -20,6 +20,7 @@ import org.springframework.stereotype.Service; import org.springframework.util.Assert; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -34,6 +35,7 @@ public class TaskMapper { private final Double defaultRamGb; private final Long defaultCpuCore; private final Boolean defaultPreemptible; + private final String defaultRegion; private MessageHelper messageHelper; private final CloudPipelineAPIClient cloudPipelineAPIClient; @@ -59,6 +61,7 @@ public TaskMapper(@Value("${cloud.pipeline.hddSize}") Integer hddSize, @Value("${cloud.pipeline.ramGb}") Double defaultRamGb, @Value("${cloud.pipeline.cpuCore}") Long defaultCpuCore, @Value("${cloud.pipeline.preemtible}") Boolean defaultPreemptible, + @Value("${cloud.pipeline.region}") String defaultRegion, CloudPipelineAPIClient cloudPipelineAPIClient, MessageHelper messageHelper) { this.defaultHddSize = hddSize; this.defaultRamGb = defaultRamGb; @@ -66,6 +69,7 @@ public TaskMapper(@Value("${cloud.pipeline.hddSize}") Integer hddSize, this.cloudPipelineAPIClient = cloudPipelineAPIClient; this.messageHelper = messageHelper; this.defaultPreemptible = defaultPreemptible; + this.defaultRegion = defaultRegion; } public PipelineStart mapToPipelineStart(TesTask tesTask) { @@ -112,8 +116,11 @@ private String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { Long cpuCores = Optional.ofNullable(tesTask.getResources()).map(TesResources::getCpuCores).orElse(defaultCpuCore); Long toolId = pipelineTool.getId(); - Long regionId = getProperRegionIdInCloudRegionsByTesZone(tesTask.getResources().getZones()); - Boolean spot = tesTask.getResources().getPreemptible(); + Long regionId = getProperRegionIdInCloudRegionsByTesZone(Optional.ofNullable(tesTask.getResources()) + .map(TesResources::getZones).orElse(Collections.singletonList(defaultRegion))); + Boolean spot = Optional.ofNullable(tesTask.getResources()) + .map(TesResources::getPreemptible).orElse(defaultPreemptible); + ; AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = cloudPipelineAPIClient .loadAllowedInstanceAndPriceTypes(toolId, regionId, spot); diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index 0754ac9455..dec9bd3638 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -5,6 +5,7 @@ cloud.pipeline.ramGb=16 cloud.pipeline.cpuCore=2 cloud.pipeline.hddSize=50 cloud.pipeline.preemtible=true +cloud.pipeline.region=eu-central-1 cloud.pipeline.service.name=CloudPipeline cloud.pipeline.doc=https://epam.github.io/cloud-pipeline/ From 0a3578309d2e35226484d9d3b25ef7199a578d4d Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Fri, 6 Sep 2019 18:34:11 +0300 Subject: [PATCH 075/194] tes-support merge into f_2803 --- .../epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java index 02c53ea2af..3bf61e6b2f 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java @@ -1,6 +1,5 @@ package com.epam.pipeline.tesadapter.service; - import com.epam.pipeline.tesadapter.common.MessageHelper; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; From 9adad92e83fd18ecec20bdb068b6898e73e2a4a6 Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Fri, 6 Sep 2019 19:23:58 +0300 Subject: [PATCH 076/194] add @NoArgsConstructor @AllArgsConstructor to entities with @Builder --- .../java/com/epam/pipeline/tesadapter/entity/TesExecutor.java | 4 ++++ .../java/com/epam/pipeline/tesadapter/entity/TesTask.java | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java index 6197daa842..73ec3507f3 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutor.java @@ -3,14 +3,18 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import java.util.List; import java.util.Map; @Data @Builder +@NoArgsConstructor +@AllArgsConstructor @ApiModel(description = "Executor describes a command to be executed, and its environment.") public class TesExecutor { @ApiModelProperty(value = "Name of the container image, for example: " + diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java index 926dc2f3bd..54e23aa932 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java @@ -3,8 +3,10 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import java.util.List; import java.util.Map; @@ -12,6 +14,8 @@ @ApiModel(description = "Task describes an instance of a task.") @Data @Builder +@NoArgsConstructor +@AllArgsConstructor public class TesTask { @ApiModelProperty(value = "Task identifier assigned by the server.") @JsonProperty("id") From 9c4386bd7d0b141108fcec30d46fcf0ffbc75e8e Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Fri, 6 Sep 2019 23:35:55 +0300 Subject: [PATCH 077/194] fixed missed brace { after merging --- .../java/com/epam/pipeline/tesadapter/service/TaskMapper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 10b17e6753..8da9158499 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -191,7 +191,8 @@ private Double convertMemoryUnitTypeToGiB(String memoryUnit) { } } return GIB_TO_GIB; - + } + public TesTask mapToTesTask(PipelineRun run) { return TesTask.builder() .id(String.valueOf(run.getId())) From 4fd8a524673a336f50837bc569c55b2572c83e83 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Mon, 9 Sep 2019 20:12:46 +0300 Subject: [PATCH 078/194] added createTesStatus not working. (Error 404) --- .../client/pipeline/CloudPipelineAPI.java | 4 ++++ .../service/CloudPipelineAPIClient.java | 5 +++++ .../tesadapter/service/TaskMapper.java | 19 +++++++++++++++---- .../service/TesTaskServiceImpl.java | 1 + 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java index 6850ee47a5..d14a3b2bbe 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java @@ -30,6 +30,7 @@ import com.epam.pipeline.entity.pipeline.Folder; import com.epam.pipeline.entity.pipeline.Pipeline; import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.entity.pipeline.PipelineTask; import com.epam.pipeline.entity.pipeline.Revision; import com.epam.pipeline.entity.pipeline.RunInstance; import com.epam.pipeline.entity.pipeline.RunLog; @@ -196,4 +197,7 @@ Call>> loadRepositoryContent(@Path(ID) Long id, //Notification methods @POST("notification/message") Call> createNotification(@Body NotificationMessageVO notification); + + @GET("/run/{runId}/tasks") + Call>> loadPipelineTasks(@Path(RUN_ID) Long runId); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index 06cf5aa21c..da03ca2f10 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -4,6 +4,7 @@ import com.epam.pipeline.client.pipeline.CloudPipelineApiBuilder; import com.epam.pipeline.entity.datastorage.AbstractDataStorage; import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.entity.pipeline.PipelineTask; import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.utils.QueryUtils; import com.epam.pipeline.vo.RunStatusVO; @@ -39,5 +40,9 @@ public List loadAllDataStorages(){ return QueryUtils.execute(cloudPipelineAPI.loadAllDataStorages()); } + public List loadPipelineTasks(final Long id){ + return QueryUtils.execute(cloudPipelineAPI.loadPipelineTasks(id)); + } + } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 5451bc6bb7..8669bb8b69 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -3,7 +3,7 @@ import com.epam.pipeline.entity.configuration.ExecutionEnvironment; import com.epam.pipeline.entity.configuration.PipeConfValueVO; import com.epam.pipeline.entity.pipeline.PipelineRun; -import com.epam.pipeline.entity.pipeline.TaskStatus; +import com.epam.pipeline.entity.pipeline.PipelineTask; import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter; import com.epam.pipeline.tesadapter.common.MessageConstants; @@ -34,6 +34,8 @@ public class TaskMapper { private final Integer defaultHddSize; @Autowired private MessageHelper messageHelper; + @Autowired + private CloudPipelineAPIClient cloudPipelineAPIClient; private static final String SEPARATOR = " "; private static final String INPUT_TYPE = "input"; private static final String OUTPUT_TYPE = "output"; @@ -78,18 +80,27 @@ private TesExecutor getExecutorFromTesExecutorsList(List tesExecuto public TesTask mapToTesTask(PipelineRun run) { return TesTask.builder() .id(String.valueOf(run.getId())) - .state(createTesState(run.getStatus())) .name(run.getPodId()) .resources(createTesResources(run)) .executors(createListExecutor(run)) .inputs(createTesInput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) .outputs(createTesOutput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) .creationTime(run.getStartDate().toString()) + .state(createTesState(run)) .build(); } - private TesState createTesState(TaskStatus status) { - switch (status) { + private TesState createTesState(PipelineRun run) { + List pipelineTaskList = cloudPipelineAPIClient.loadPipelineTasks(run.getId()); + for(PipelineTask p: pipelineTaskList){ + if((p.getName().equalsIgnoreCase("Console")) && (pipelineTaskList.size() == 1)){ + return TesState.QUEUED; + } else if( (p.getName().equalsIgnoreCase("Console")) && (pipelineTaskList.size() > 1) && + (!p.getName().equalsIgnoreCase("InitializeEnvironment"))){ + return TesState.INITIALIZING; + } + } + switch (run.getStatus()) { case RUNNING: return TesState.RUNNING; case PAUSED: diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index a1358d8980..98942d8f61 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -2,6 +2,7 @@ import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.entity.pipeline.PipelineTask; import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.tesadapter.common.MessageConstants; import com.epam.pipeline.tesadapter.common.MessageHelper; From f665158d96cddaaba367715745ebbba6dd41e4f1 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 10 Sep 2019 18:47:18 +0300 Subject: [PATCH 079/194] added in method createTesStatus statuses Console, Initialization --- .../pipeline/client/pipeline/CloudPipelineAPI.java | 2 +- .../epam/pipeline/tesadapter/entity/TesResources.java | 4 ++++ .../com/epam/pipeline/tesadapter/entity/TesTask.java | 4 ++++ .../epam/pipeline/tesadapter/service/TaskMapper.java | 11 +++++++---- .../tesadapter/service/TesTaskServiceImpl.java | 3 +-- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java index d14a3b2bbe..edc68e8086 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java @@ -198,6 +198,6 @@ Call>> loadRepositoryContent(@Path(ID) Long id, @POST("notification/message") Call> createNotification(@Body NotificationMessageVO notification); - @GET("/run/{runId}/tasks") + @GET("run/{runId}/tasks") Call>> loadPipelineTasks(@Path(RUN_ID) Long runId); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java index c45fd1f4c5..1d3c1544a8 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java @@ -3,8 +3,10 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import javax.validation.Valid; import java.util.List; @@ -12,6 +14,8 @@ @ApiModel(description = "Resources describes the resources requested by a task.") @Data @Builder +@NoArgsConstructor +@AllArgsConstructor public class TesResources { @ApiModelProperty(value = "Requested number of CPUs") @JsonProperty("cpu_cores") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java index 926dc2f3bd..54e23aa932 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTask.java @@ -3,8 +3,10 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import java.util.List; import java.util.Map; @@ -12,6 +14,8 @@ @ApiModel(description = "Task describes an instance of a task.") @Data @Builder +@NoArgsConstructor +@AllArgsConstructor public class TesTask { @ApiModelProperty(value = "Task identifier assigned by the server.") @JsonProperty("id") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 8669bb8b69..9e7e8d1e0d 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -92,11 +92,14 @@ public TesTask mapToTesTask(PipelineRun run) { private TesState createTesState(PipelineRun run) { List pipelineTaskList = cloudPipelineAPIClient.loadPipelineTasks(run.getId()); - for(PipelineTask p: pipelineTaskList){ - if((p.getName().equalsIgnoreCase("Console")) && (pipelineTaskList.size() == 1)){ + for (PipelineTask p : pipelineTaskList) { + if (p.getName().equalsIgnoreCase("Console") && pipelineTaskList.size() == 1) { return TesState.QUEUED; - } else if( (p.getName().equalsIgnoreCase("Console")) && (pipelineTaskList.size() > 1) && - (!p.getName().equalsIgnoreCase("InitializeEnvironment"))){ + } else if (p.getName().equalsIgnoreCase("InitializeEnvironment")) { + break; + } else if ((p.getName().equalsIgnoreCase("Console") && + !p.getName().equalsIgnoreCase("InitializeEnvironment")) + && pipelineTaskList.size() > 1) { return TesState.INITIALIZING; } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 98942d8f61..d8eccfcfb5 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -2,7 +2,6 @@ import com.epam.pipeline.entity.pipeline.PipelineRun; -import com.epam.pipeline.entity.pipeline.PipelineTask; import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.tesadapter.common.MessageConstants; import com.epam.pipeline.tesadapter.common.MessageHelper; @@ -13,8 +12,8 @@ import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.vo.RunStatusVO; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; From a604b5e3191c7ecd164d7907ec1169ae08df04b9 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 10 Sep 2019 21:06:43 +0300 Subject: [PATCH 080/194] add import logs from pipeline to tes --- .../tesadapter/entity/TesExecutorLog.java | 6 ++++ .../tesadapter/entity/TesTaskLog.java | 7 +++++ .../service/CloudPipelineAPIClient.java | 5 ++++ .../tesadapter/service/TaskMapper.java | 28 ++++++++++++++++++- 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java index a5bd614b3b..4d02154932 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesExecutorLog.java @@ -3,10 +3,16 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; @Data @ApiModel(description = "ExecutorLog describes logging information related to an Executor.") +@Builder +@NoArgsConstructor +@AllArgsConstructor public class TesExecutorLog { @ApiModelProperty(value = "Time the executor started, in RFC 3339 format.") @JsonProperty("start_time") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java index b7dd72e7f3..9a35b03ae5 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTaskLog.java @@ -3,12 +3,19 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; + import java.util.List; import java.util.Map; @ApiModel(description = "TaskLog describes logging information related to a Task.") @Data +@Builder +@AllArgsConstructor +@NoArgsConstructor public class TesTaskLog { @ApiModelProperty(value = "Logs for each executor") @JsonProperty("logs") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index 06cf5aa21c..8d9ffae0ac 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -4,6 +4,7 @@ import com.epam.pipeline.client.pipeline.CloudPipelineApiBuilder; import com.epam.pipeline.entity.datastorage.AbstractDataStorage; import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.entity.pipeline.RunLog; import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.utils.QueryUtils; import com.epam.pipeline.vo.RunStatusVO; @@ -39,5 +40,9 @@ public List loadAllDataStorages(){ return QueryUtils.execute(cloudPipelineAPI.loadAllDataStorages()); } + public List getRunLog(final Long runId){ + return QueryUtils.execute(cloudPipelineAPI.loadLogs(runId)); + } + } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 5451bc6bb7..60c926285d 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -3,17 +3,20 @@ import com.epam.pipeline.entity.configuration.ExecutionEnvironment; import com.epam.pipeline.entity.configuration.PipeConfValueVO; import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.entity.pipeline.RunLog; import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter; import com.epam.pipeline.tesadapter.common.MessageConstants; import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.entity.TesExecutor; +import com.epam.pipeline.tesadapter.entity.TesExecutorLog; import com.epam.pipeline.tesadapter.entity.TesInput; import com.epam.pipeline.tesadapter.entity.TesOutput; import com.epam.pipeline.tesadapter.entity.TesResources; import com.epam.pipeline.tesadapter.entity.TesState; import com.epam.pipeline.tesadapter.entity.TesTask; +import com.epam.pipeline.tesadapter.entity.TesTaskLog; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.ListUtils; import org.apache.commons.collections4.MapUtils; @@ -22,6 +25,7 @@ import org.springframework.stereotype.Service; import org.springframework.util.Assert; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -34,6 +38,7 @@ public class TaskMapper { private final Integer defaultHddSize; @Autowired private MessageHelper messageHelper; + private CloudPipelineAPIClient cloudPipelineAPIClient; private static final String SEPARATOR = " "; private static final String INPUT_TYPE = "input"; private static final String OUTPUT_TYPE = "output"; @@ -85,6 +90,7 @@ public TesTask mapToTesTask(PipelineRun run) { .inputs(createTesInput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) .outputs(createTesOutput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) .creationTime(run.getStartDate().toString()) + .logs(createTesTaskLog(run.getId())) .build(); } @@ -134,10 +140,30 @@ private TesResources createTesResources(PipelineRun run){ .build(); } - private List createListExecutor(PipelineRun run){ + private List createListExecutor(PipelineRun run) { return ListUtils.emptyIfNull(Arrays.asList(TesExecutor.builder() .command(ListUtils.emptyIfNull(Arrays.asList(run.getActualCmd().split(SEPARATOR)))) .env(run.getEnvVars()) .build())); } + + private List createTesTaskLog(final Long runId) { + List runLogList = ListUtils.emptyIfNull(cloudPipelineAPIClient.getRunLog(runId)); + List tesExecutorLogList = new ArrayList(); + tesExecutorLogList.add(TesExecutorLog.builder() + .stdout(runLogList.stream().map(RunLog::getDate).findAny().toString() + + runLogList.stream().map(RunLog::getTaskName).iterator().next() + + runLogList.stream().map(RunLog::getLogText).iterator().next()) + .build()); + List tesTaskLogs = new ArrayList<>(); + tesTaskLogs.add(TesTaskLog.builder() + .logs(tesExecutorLogList) + .build()); + return ListUtils.emptyIfNull(tesTaskLogs); + } + + @Autowired + public void setCloudPipelineAPIClient(CloudPipelineAPIClient cloudPipelineAPIClient) { + this.cloudPipelineAPIClient = cloudPipelineAPIClient; + } } From 6e60fa94832ded845363199f4deff05bf9be947d Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 10 Sep 2019 22:30:37 +0300 Subject: [PATCH 081/194] added couple unit-tests for TaskMapperTest class. Not finished yet. upgrade build.gradle to Mockito 2 to be able to mock "final" classes. --- tes-adapter/build.gradle | 3 +- .../tesadapter/entity/TesResources.java | 6 ++- .../tesadapter/service/TaskMapper.java | 27 +++++++---- .../tesadapter/service/TaskMapperTest.java | 48 +++++++++++++++++++ 4 files changed, 72 insertions(+), 12 deletions(-) diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index 17498e388c..3a30cf6ed6 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -61,8 +61,7 @@ dependencies { testImplementation('org.junit.jupiter:junit-jupiter-api:5.2.0') testCompile('org.junit.jupiter:junit-jupiter-params:5.2.0') testRuntime('org.junit.jupiter:junit-jupiter-engine:5.2.0') - testCompile "org.mockito:mockito-core:2.+" - testCompile('org.mockito:mockito-junit-jupiter:2.18.3') + testImplementation 'org.mockito:mockito-inline:2.13.0' testImplementation('org.springframework.boot:spring-boot-starter-test') { exclude group: 'junit', module: 'junit' //direct exclusion junit4 } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java index c45fd1f4c5..812da16e0c 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java @@ -3,15 +3,19 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import javax.validation.Valid; import java.util.List; -@ApiModel(description = "Resources describes the resources requested by a task.") @Data @Builder +@NoArgsConstructor +@AllArgsConstructor +@ApiModel(description = "Resources describes the resources requested by a task.") public class TesResources { @ApiModelProperty(value = "Requested number of CPUs") @JsonProperty("cpu_cores") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 8da9158499..98a00e2b1e 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -88,20 +88,23 @@ public PipelineStart mapToPipelineStart(TesTask tesTask) { PipelineStart pipelineStart = new PipelineStart(); Map params = new HashMap<>(); TesExecutor tesExecutor = getExecutorFromTesExecutorsList(tesTask.getExecutors()); - - Assert.notNull(tesExecutor.getImage(), messageHelper.getMessage( MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, IMAGE)); Tool pipelineTool = loadToolByTesImage(tesExecutor.getImage()); - pipelineStart.setInstanceType(getProperInstanceType(tesTask, pipelineTool)); pipelineStart.setCmdTemplate(String.join(SEPARATOR, tesExecutor.getCommand())); pipelineStart.setDockerImage(tesExecutor.getImage()); pipelineStart.setExecutionEnvironment(ExecutionEnvironment.CLOUD_PLATFORM); pipelineStart.setHddSize(Optional.ofNullable(tesTask.getResources()) - .map(tesResources -> tesResources.getDiskGb().intValue()).orElse(defaultHddSize)); + .map(tesResources -> { + if (Optional.ofNullable(tesResources.getDiskGb()).isPresent()) { + return tesResources.getDiskGb().intValue(); + } + return defaultHddSize; + }).orElse(defaultHddSize)); pipelineStart.setIsSpot(Optional.ofNullable(tesTask.getResources()) - .map(TesResources::getPreemptible).orElse(defaultPreemptible)); + .map(tesResources -> Optional.ofNullable(tesResources.getPreemptible()).orElse(defaultPreemptible)) + .orElse(defaultPreemptible)); pipelineStart.setForce(false); pipelineStart.setNonPause(true); ListUtils.emptyIfNull(tesTask.getInputs()).forEach(tesInput -> @@ -121,14 +124,20 @@ private TesExecutor getExecutorFromTesExecutorsList(List tesExecuto } - private String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { + public String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { Double ramGb = - Optional.ofNullable(tesTask.getResources()).map(TesResources::getRamGb).orElse(defaultRamGb); + Optional.ofNullable(tesTask.getResources()) + .map(tesResources -> Optional.ofNullable(tesResources.getRamGb()).orElse(defaultRamGb)) + .orElse(defaultRamGb); Long cpuCores = - Optional.ofNullable(tesTask.getResources()).map(TesResources::getCpuCores).orElse(defaultCpuCore); + Optional.ofNullable(tesTask.getResources()) + .map(tesResources -> Optional.ofNullable(tesResources.getCpuCores()).orElse(defaultCpuCore)) + .orElse(defaultCpuCore); Long toolId = pipelineTool.getId(); Long regionId = getProperRegionIdInCloudRegionsByTesZone(Optional.ofNullable(tesTask.getResources()) - .map(TesResources::getZones).orElse(Collections.singletonList(defaultRegion))); + .map(tesResources -> Optional.ofNullable(tesResources.getZones()) + .orElse(Collections.singletonList(defaultRegion))) + .orElse(Collections.singletonList(defaultRegion))); Boolean spot = Optional.ofNullable(tesTask.getResources()) .map(TesResources::getPreemptible).orElse(defaultPreemptible); ; diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index a14b946ee2..133a92129e 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -9,4 +9,52 @@ class TaskMapperTest { void mapToPipelineStartTest() { } + + @ParameterizedTest + @MethodSource("provideInputForGetProperInstanceTypeTest") + void getProperInstanceTypeShouldReturnProperInstanceName(String instanceTypeName, Double ramGb, Long cpuCore) { + zones.add(STUBBED_REGION_NAME); + abstractCloudRegions.add(abstractCloudRegion); + when(tesTask.getResources().getRamGb()).thenReturn(ramGb); + when(tesTask.getResources().getCpuCores()).thenReturn(cpuCore); + when(tesTask.getResources().getZones()).thenReturn(zones); + + when(allowedInstanceAndPriceTypes.getAllowedInstanceTypes()).thenReturn(allowedInstanceTypes); + fillWithAllowedInstanceTypes(allowedInstanceTypes); + assertEquals(instanceTypeName, taskMapper.getProperInstanceType(tesTask, tool)); + } + + private static Stream provideInputForGetProperInstanceTypeTest() { + return Stream.of( + Arguments.of("m5.large", 8.0, 2L), + Arguments.of("c5.2xlarge ", 16.0, 8L), + Arguments.of("c5.4xlarge", 32.0, 16L), + Arguments.of("m5.4xlarge", 64.0, 16L), + Arguments.of("m5.8xlarge", 128.0, 32L), + Arguments.of("m5.16xlarge", 256.0, 64L), + Arguments.of("c5.18xlarge", 144.0, 72L), + Arguments.of("m5.24xlarge", 384.0, 96L) + ); + } + + private void fillWithAllowedInstanceTypes(List instanceTypeList) { + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.large", 8.0, 2L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.2xlarge ", 16.0, 8L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.4xlarge", 32.0, 16L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.4xlarge", 64.0, 16L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.8xlarge", 128.0, 32L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.16xlarge", 256.0, 64L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.18xlarge", 144.0, 72L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.24xlarge", 192.0, 96L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.24xlarge", 384.0, 96L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("p2.16xlarge", 768.0, 64L)); + } + + private InstanceType getInstanceWithNameRamAndCpu(String instanceName, Double instanceRam, Long instanceCpu) { + InstanceType instanceType = new InstanceType(); + instanceType.setName(instanceName); + instanceType.setMemory(instanceRam.floatValue()); + instanceType.setVCPU(instanceCpu.intValue()); + return instanceType; + } } \ No newline at end of file From b9fdfaf037df84061cd89507ae272fe424ad46d5 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 10 Sep 2019 22:31:53 +0300 Subject: [PATCH 082/194] added simple unit-test for TaskMapper class. Need upgrade. --- .../tesadapter/service/TaskMapper.java | 22 ++--- .../tesadapter/service/TaskMapperTest.java | 85 ++++++++++++++++++- 2 files changed, 91 insertions(+), 16 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 98a00e2b1e..cb7dfb7e73 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -4,9 +4,9 @@ import com.epam.pipeline.entity.cluster.InstanceType; import com.epam.pipeline.entity.configuration.ExecutionEnvironment; import com.epam.pipeline.entity.configuration.PipeConfValueVO; -import com.epam.pipeline.entity.pipeline.Tool; import com.epam.pipeline.entity.pipeline.PipelineRun; import com.epam.pipeline.entity.pipeline.TaskStatus; +import com.epam.pipeline.entity.pipeline.Tool; import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter; import com.epam.pipeline.tesadapter.common.MessageConstants; @@ -26,12 +26,9 @@ import org.springframework.stereotype.Service; import org.springframework.util.Assert; - +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; - -import java.util.Arrays; - import java.util.HashMap; import java.util.List; import java.util.Map; @@ -117,7 +114,7 @@ public PipelineStart mapToPipelineStart(TesTask tesTask) { return pipelineStart; } - private TesExecutor getExecutorFromTesExecutorsList(List tesExecutors) { + public TesExecutor getExecutorFromTesExecutorsList(List tesExecutors) { Assert.isTrue(tesExecutors.size() == ONLY_ONE, messageHelper.getMessage( MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, EXECUTORS, tesExecutors)); return tesExecutors.get(FIRST); @@ -139,9 +136,8 @@ public String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { .orElse(Collections.singletonList(defaultRegion))) .orElse(Collections.singletonList(defaultRegion))); Boolean spot = Optional.ofNullable(tesTask.getResources()) - .map(TesResources::getPreemptible).orElse(defaultPreemptible); - ; - + .map(tesResources -> Optional.ofNullable(tesResources.getPreemptible()).orElse(defaultPreemptible)) + .orElse(defaultPreemptible); AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = cloudPipelineAPIClient .loadAllowedInstanceAndPriceTypes(toolId, regionId, spot); Assert.notEmpty(allowedInstanceAndPriceTypes.getAllowedInstanceTypes(), messageHelper.getMessage( @@ -170,7 +166,7 @@ private String evaluateMostProperInstanceType(AllowedInstanceAndPriceTypes allow } /** - * Calculates the effective coefficient for {@code instanceType}.The result + * Calculates the coefficient of deviation for {@code instanceType}.The result * depends on the level of difference between the used values {@code memory} and * {@code vCPU} in {@code instanceType} and the entered values {@code ramGb} and * {@code cpuCores}, respectively. From greater difference, comes greater coefficient. @@ -201,7 +197,7 @@ private Double convertMemoryUnitTypeToGiB(String memoryUnit) { } return GIB_TO_GIB; } - + public TesTask mapToTesTask(PipelineRun run) { return TesTask.builder() .id(String.valueOf(run.getId())) @@ -254,14 +250,14 @@ private List createTesOutput(List parameters) { return ListUtils.emptyIfNull(Arrays.asList(tesOutput)); } - private TesResources createTesResources(PipelineRun run){ + private TesResources createTesResources(PipelineRun run) { return TesResources.builder() .preemptible(run.getInstance().getSpot()) .diskGb(new Double(run.getInstance().getNodeDisk())) .build(); } - private List createListExecutor(PipelineRun run){ + private List createListExecutor(PipelineRun run) { return ListUtils.emptyIfNull(Arrays.asList(TesExecutor.builder() .command(ListUtils.emptyIfNull(Arrays.asList(run.getActualCmd().split(SEPARATOR)))) .env(run.getEnvVars()) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index 133a92129e..585a631089 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -1,13 +1,92 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.entity.cluster.AllowedInstanceAndPriceTypes; +import com.epam.pipeline.entity.cluster.InstanceType; +import com.epam.pipeline.entity.pipeline.Tool; +import com.epam.pipeline.entity.region.AbstractCloudRegion; +import com.epam.pipeline.tesadapter.common.MessageConstants; +import com.epam.pipeline.tesadapter.common.MessageHelper; +import com.epam.pipeline.tesadapter.configuration.AppConfiguration; +import com.epam.pipeline.tesadapter.entity.TesExecutor; +import com.epam.pipeline.tesadapter.entity.TesResources; +import com.epam.pipeline.tesadapter.entity.TesTask; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; -import static org.junit.jupiter.api.Assertions.*; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + + +@ExtendWith(value = SpringExtension.class) +@ContextConfiguration(classes = {AppConfiguration.class}) +@SuppressWarnings("unused") class TaskMapperTest { - @Test - void mapToPipelineStartTest() { + private static final String EXECUTORS = "executors"; + private static final Integer STUBBED_HDD_SIZE = 50; + private static final Double STUBBED_RAM_GB = 16.0; + private static final Long STUBBED_CPU_CORES = 2L; + private static final Boolean STUBBED_PREEMPTIBLE = true; + private static final String STUBBED_REGION_NAME = "eu-central-1"; + private static final Long STUBBED_REGION_ID = 1L; + private static final Long STUBBED_TOOL_ID = 11584L; + private static final String IMAGE = "cp-docker-registry.default.svc.cluster.local:31443/library/centos:latest"; + + @Autowired + private MessageHelper messageHelper; + + private TaskMapper taskMapper; + + + private List zones = new ArrayList<>(); + private List abstractCloudRegions = new ArrayList<>(); + private List allowedInstanceTypes = new ArrayList<>(); + + private AbstractCloudRegion abstractCloudRegion = mock(AbstractCloudRegion.class); + private CloudPipelineAPIClient cloudPipelineAPIClient = mock(CloudPipelineAPIClient.class); + private AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = mock(AllowedInstanceAndPriceTypes.class); + private Tool tool = mock(Tool.class); + private TesTask tesTask = mock(TesTask.class); + + + @BeforeEach + void setUp() { + when(abstractCloudRegion.getName()).thenReturn(STUBBED_REGION_NAME); + when(abstractCloudRegion.getId()).thenReturn(STUBBED_REGION_ID); + when(cloudPipelineAPIClient.loadAllRegions()).thenReturn(abstractCloudRegions); + when(cloudPipelineAPIClient.loadTool(IMAGE)).thenReturn(tool); + when(cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(STUBBED_TOOL_ID, + STUBBED_REGION_ID, STUBBED_PREEMPTIBLE)).thenReturn(allowedInstanceAndPriceTypes); + when(tool.getId()).thenReturn(STUBBED_TOOL_ID); + when(tesTask.getResources()).thenReturn(mock(TesResources.class)); + when(tesTask.getResources().getPreemptible()).thenReturn(true); + + this.taskMapper = new TaskMapper(STUBBED_HDD_SIZE, STUBBED_RAM_GB, STUBBED_CPU_CORES, + STUBBED_PREEMPTIBLE, STUBBED_REGION_NAME, cloudPipelineAPIClient, this.messageHelper); + } + @Test() + void expectIllegalArgExceptionWhenRunGetExecutorFromTesExecutorsListTest() { + List tesExecutors = new ArrayList<>(); + tesExecutors.add(new TesExecutor()); + tesExecutors.add(new TesExecutor()); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> taskMapper.getExecutorFromTesExecutorsList(tesExecutors)); + assertTrue(exception.getMessage().contains(messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, EXECUTORS, tesExecutors))); } @ParameterizedTest From 6608f4f5afc17e045d21485c954bc956e7be742d Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 11 Sep 2019 15:18:32 +0300 Subject: [PATCH 083/194] added more unit-tests in TaskMapperTest, couple refactoring in TaskMapper for more safety running --- .../tesadapter/service/TaskMapper.java | 12 +- .../tesadapter/service/TaskMapperTest.java | 153 ++++++++++++++---- 2 files changed, 127 insertions(+), 38 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index cb7dfb7e73..e443925d95 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -145,11 +145,13 @@ public String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { return evaluateMostProperInstanceType(allowedInstanceAndPriceTypes, ramGb, cpuCores); } - private Tool loadToolByTesImage(String image) { - return cloudPipelineAPIClient.loadTool(image); + public Tool loadToolByTesImage(String image) { + Assert.hasText(image, messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, image)); + return Optional.ofNullable(cloudPipelineAPIClient.loadTool(image)).orElseThrow(IllegalArgumentException::new); } - private Long getProperRegionIdInCloudRegionsByTesZone(List zones) { + public Long getProperRegionIdInCloudRegionsByTesZone(List zones) { Assert.isTrue(zones.size() == ONLY_ONE, messageHelper.getMessage( MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, ZONES, zones)); return cloudPipelineAPIClient.loadAllRegions().stream().filter( @@ -177,8 +179,8 @@ private String evaluateMostProperInstanceType(AllowedInstanceAndPriceTypes allow * @return double - correspond coefficient for {@code instanceType} */ private Double calculateInstanceCoef(InstanceType instanceType, Double ramGb, Long cpuCores) { - return Math.abs((convertMemoryUnitTypeToGiB(instanceType.getMemoryUnit()) * instanceType.getMemory() - / ramGb + (double) instanceType.getVCPU() / (double) cpuCores) / 2 - 1); + return Math.abs(convertMemoryUnitTypeToGiB(instanceType.getMemoryUnit()) * instanceType.getMemory() + - ramGb) / ramGb + Math.abs((double) (instanceType.getVCPU() - cpuCores)) / cpuCores; } private Double convertMemoryUnitTypeToGiB(String memoryUnit) { diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index 585a631089..75ee096666 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -10,6 +10,7 @@ import com.epam.pipeline.tesadapter.entity.TesExecutor; import com.epam.pipeline.tesadapter.entity.TesResources; import com.epam.pipeline.tesadapter.entity.TesTask; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -35,25 +36,24 @@ @ContextConfiguration(classes = {AppConfiguration.class}) @SuppressWarnings("unused") class TaskMapperTest { + private static final Integer DEFAULT_HDD_SIZE = 50; + private static final Double DEFAULT_RAM_GB = 8.0; + private static final Long DEFAULT_CPU_CORES = 2L; + private static final Boolean DEFAULT_PREEMPTIBLE = true; + private static final String DEFAULT_REGION_NAME = "eu-central-1"; private static final String EXECUTORS = "executors"; - private static final Integer STUBBED_HDD_SIZE = 50; - private static final Double STUBBED_RAM_GB = 16.0; - private static final Long STUBBED_CPU_CORES = 2L; - private static final Boolean STUBBED_PREEMPTIBLE = true; - private static final String STUBBED_REGION_NAME = "eu-central-1"; private static final Long STUBBED_REGION_ID = 1L; private static final Long STUBBED_TOOL_ID = 11584L; - private static final String IMAGE = "cp-docker-registry.default.svc.cluster.local:31443/library/centos:latest"; + private static final String STUBBED_IMAGE = "cp-docker-registry.default.svc.cluster.local:31443/library/centos:latest"; + private static List allowedInstanceTypes; + @Autowired private MessageHelper messageHelper; private TaskMapper taskMapper; - - private List zones = new ArrayList<>(); private List abstractCloudRegions = new ArrayList<>(); - private List allowedInstanceTypes = new ArrayList<>(); private AbstractCloudRegion abstractCloudRegion = mock(AbstractCloudRegion.class); private CloudPipelineAPIClient cloudPipelineAPIClient = mock(CloudPipelineAPIClient.class); @@ -61,25 +61,31 @@ class TaskMapperTest { private Tool tool = mock(Tool.class); private TesTask tesTask = mock(TesTask.class); + @BeforeAll + static void setUpAll() { + allowedInstanceTypes = new ArrayList<>(); + fillWithAllowedInstanceTypes(allowedInstanceTypes); + } @BeforeEach void setUp() { - when(abstractCloudRegion.getName()).thenReturn(STUBBED_REGION_NAME); + when(abstractCloudRegion.getName()).thenReturn(DEFAULT_REGION_NAME); when(abstractCloudRegion.getId()).thenReturn(STUBBED_REGION_ID); when(cloudPipelineAPIClient.loadAllRegions()).thenReturn(abstractCloudRegions); - when(cloudPipelineAPIClient.loadTool(IMAGE)).thenReturn(tool); + when(cloudPipelineAPIClient.loadTool(STUBBED_IMAGE)).thenReturn(tool); when(cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(STUBBED_TOOL_ID, - STUBBED_REGION_ID, STUBBED_PREEMPTIBLE)).thenReturn(allowedInstanceAndPriceTypes); + STUBBED_REGION_ID, DEFAULT_PREEMPTIBLE)).thenReturn(allowedInstanceAndPriceTypes); when(tool.getId()).thenReturn(STUBBED_TOOL_ID); when(tesTask.getResources()).thenReturn(mock(TesResources.class)); when(tesTask.getResources().getPreemptible()).thenReturn(true); - - this.taskMapper = new TaskMapper(STUBBED_HDD_SIZE, STUBBED_RAM_GB, STUBBED_CPU_CORES, - STUBBED_PREEMPTIBLE, STUBBED_REGION_NAME, cloudPipelineAPIClient, this.messageHelper); + zones.add(DEFAULT_REGION_NAME); + abstractCloudRegions.add(abstractCloudRegion); + this.taskMapper = new TaskMapper(DEFAULT_HDD_SIZE, DEFAULT_RAM_GB, DEFAULT_CPU_CORES, + DEFAULT_PREEMPTIBLE, DEFAULT_REGION_NAME, cloudPipelineAPIClient, this.messageHelper); } @Test() - void expectIllegalArgExceptionWhenRunGetExecutorFromTesExecutorsListTest() { + void expectIllegalArgExceptionWhenRunGetExecutorFromTesExecutorsList() { List tesExecutors = new ArrayList<>(); tesExecutors.add(new TesExecutor()); tesExecutors.add(new TesExecutor()); @@ -92,44 +98,125 @@ void expectIllegalArgExceptionWhenRunGetExecutorFromTesExecutorsListTest() { @ParameterizedTest @MethodSource("provideInputForGetProperInstanceTypeTest") void getProperInstanceTypeShouldReturnProperInstanceName(String instanceTypeName, Double ramGb, Long cpuCore) { - zones.add(STUBBED_REGION_NAME); - abstractCloudRegions.add(abstractCloudRegion); when(tesTask.getResources().getRamGb()).thenReturn(ramGb); when(tesTask.getResources().getCpuCores()).thenReturn(cpuCore); when(tesTask.getResources().getZones()).thenReturn(zones); - when(allowedInstanceAndPriceTypes.getAllowedInstanceTypes()).thenReturn(allowedInstanceTypes); - fillWithAllowedInstanceTypes(allowedInstanceTypes); assertEquals(instanceTypeName, taskMapper.getProperInstanceType(tesTask, tool)); } + @Test + void expectIllegalArgExceptionWhenRunGetProperRegionIdInCloudRegionsByTesZone() { + zones.add("Second zone"); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> taskMapper.getProperRegionIdInCloudRegionsByTesZone(zones)); + assertTrue(exception.getMessage().contains(messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, EXECUTORS, zones))); + } + + @Test + void getProperRegionIdInCloudRegionsByTesZoneShouldReturnRegionId() { + assertEquals(STUBBED_REGION_ID, taskMapper.getProperRegionIdInCloudRegionsByTesZone(zones)); + } + + @Test + void expectIllegalArgExceptionWhenRunLoadToolByTesImageWithEmptyOrNullImage() { + IllegalArgumentException exception1 = assertThrows(IllegalArgumentException.class, + () -> taskMapper.loadToolByTesImage("")); + assertTrue(exception1.getMessage().contains(messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, ""))); + + IllegalArgumentException exception2 = assertThrows(IllegalArgumentException.class, + () -> taskMapper.loadToolByTesImage(null)); + assertTrue(exception2.getMessage().contains(messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, null))); + } + + @Test + void loadToolByTesImageShouldReturnToolByRequestingImage() { + assertEquals(tool, taskMapper.loadToolByTesImage(STUBBED_IMAGE)); + } + private static Stream provideInputForGetProperInstanceTypeTest() { return Stream.of( - Arguments.of("m5.large", 8.0, 2L), - Arguments.of("c5.2xlarge ", 16.0, 8L), + //Perfect cases with 0.0 deviation + Arguments.of("c5.12xlarge", 96.0, 48L), + Arguments.of("c5.18xlarge", 144.0, 72L), + Arguments.of("c5.24xlarge", 192.0, 96L), + Arguments.of("c5.2xlarge", 16.0, 8L), Arguments.of("c5.4xlarge", 32.0, 16L), + Arguments.of("c5.9xlarge", 72.0, 36L), + Arguments.of("c5.large", 4.0, 2L), +// Arguments.of("c5.metal", 192.0, 96L), duplicate with "c5.24xlarge", 192.0, 96L + Arguments.of("c5.xlarge", 8.0, 4L), + Arguments.of("m5.12xlarge", 192.0, 48L), + Arguments.of("m5.16xlarge", 256.0, 64L), + Arguments.of("m5.24xlarge", 384.0, 96L), + Arguments.of("m5.2xlarge", 32.0, 8L), Arguments.of("m5.4xlarge", 64.0, 16L), Arguments.of("m5.8xlarge", 128.0, 32L), - Arguments.of("m5.16xlarge", 256.0, 64L), - Arguments.of("c5.18xlarge", 144.0, 72L), - Arguments.of("m5.24xlarge", 384.0, 96L) + Arguments.of("m5.large", 8.0, 2L), +// Arguments.of("m5.metal", 384.0, 96L), duplicate with "m5.24xlarge", 384.0, 96L + Arguments.of("m5.xlarge", 16.0, 4L), + Arguments.of("p2.16xlarge", 768.0, 64L), + Arguments.of("p2.8xlarge", 488.0, 32L), + Arguments.of("p2.xlarge", 61.0, 4L), + Arguments.of("r5.12xlarge", 384.0, 48L), + Arguments.of("r5.16xlarge", 512.0, 64L), + Arguments.of("r5.24xlarge", 768.0, 96L), + Arguments.of("r5.2xlarge", 64.0, 8L), + Arguments.of("r5.4xlarge", 128.0, 16L), + Arguments.of("r5.8xlarge", 256.0, 32L), + Arguments.of("r5.large", 16.0, 2L), + Arguments.of("r5.xlarge", 32.0, 4L), + + //Different cases with non 0.0 deviation + Arguments.of("m5.24xlarge", 384.0, 110L), //"m5.24xlarge", 384.0, 96L has closest coef. + Arguments.of("r5.2xlarge", 150.0, 8L), //"r5.2xlarge", 64.0, 8L has closest coef. + Arguments.of("r5.4xlarge", 128.0, 10L), //"r5.4xlarge", 128.0, 16L has closest coef. + Arguments.of("p2.16xlarge", 650.0, 64L), //"p2.16xlarge", 768.0, 64L has closest coef. + Arguments.of("p2.8xlarge", 650.0, 32L), //"p2.8xlarge", 488.0, 32L has closest coef. + + //Check that the default values will be used if the request does not have them, respectively + Arguments.of("r5.large", 650.0, null), //defaultCpuCore = 2.0, closest should be "r5.large", 16.0, 2L + Arguments.of("c5.xlarge", null, 32L), //defaultRamGb = 8.0, closest should be "c5.xlarge", 8.0, 4L + Arguments.of("m5.large", null, null) //defaultRamGb = 8.0 and defaultCpuCore = 2 => "m5.large", 8.0, 2L ); } - private void fillWithAllowedInstanceTypes(List instanceTypeList) { - instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.large", 8.0, 2L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.2xlarge ", 16.0, 8L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.4xlarge", 32.0, 16L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.4xlarge", 64.0, 16L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.8xlarge", 128.0, 32L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.16xlarge", 256.0, 64L)); + private static void fillWithAllowedInstanceTypes(List instanceTypeList) { + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.12xlarge", 96.0, 48L)); instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.18xlarge", 144.0, 72L)); instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.24xlarge", 192.0, 96L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.2xlarge", 16.0, 8L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.4xlarge", 32.0, 16L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.9xlarge", 72.0, 36L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.large", 4.0, 2L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.metal", 192.0, 96L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.xlarge", 8.0, 4L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.12xlarge", 192.0, 48L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.16xlarge", 256.0, 64L)); instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.24xlarge", 384.0, 96L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.2xlarge", 32.0, 8L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.4xlarge", 64.0, 16L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.8xlarge", 128.0, 32L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.large", 8.0, 2L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.metal", 384.0, 96L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.xlarge", 16.0, 4L)); instanceTypeList.add(getInstanceWithNameRamAndCpu("p2.16xlarge", 768.0, 64L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("p2.8xlarge", 488.0, 32L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("p2.xlarge", 61.0, 4L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.12xlarge", 384.0, 48L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.16xlarge", 512.0, 64L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.24xlarge", 768.0, 96L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.2xlarge", 64.0, 8L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.4xlarge", 128.0, 16L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.8xlarge", 256.0, 32L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.large", 16.0, 2L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.xlarge", 32.0, 4L)); } - private InstanceType getInstanceWithNameRamAndCpu(String instanceName, Double instanceRam, Long instanceCpu) { + private static InstanceType getInstanceWithNameRamAndCpu(String instanceName, Double instanceRam, Long instanceCpu) { InstanceType instanceType = new InstanceType(); instanceType.setName(instanceName); instanceType.setMemory(instanceRam.floatValue()); From f5b8613ab51495a936d7dcea4c7d870d07b495a8 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 11 Sep 2019 15:27:54 +0300 Subject: [PATCH 084/194] added check-ofNullable-points for returning values in couple TasMapper class methods. --- .../com/epam/pipeline/tesadapter/service/TaskMapper.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index e443925d95..ba0140f7ec 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -154,14 +154,15 @@ public Tool loadToolByTesImage(String image) { public Long getProperRegionIdInCloudRegionsByTesZone(List zones) { Assert.isTrue(zones.size() == ONLY_ONE, messageHelper.getMessage( MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, ZONES, zones)); - return cloudPipelineAPIClient.loadAllRegions().stream().filter( + return Optional.ofNullable(cloudPipelineAPIClient.loadAllRegions().stream().filter( region -> region.getName().equalsIgnoreCase(zones.get(FIRST))) - .collect(Collectors.toList()).get(FIRST).getId(); + .collect(Collectors.toList()).get(FIRST).getId()).orElseThrow(IllegalArgumentException::new); } private String evaluateMostProperInstanceType(AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes, Double ramGb, Long cpuCores) { - return allowedInstanceAndPriceTypes.getAllowedInstanceTypes().stream() + return Optional.ofNullable(allowedInstanceAndPriceTypes.getAllowedInstanceTypes()) + .orElseThrow(IllegalArgumentException::new).stream() .min(Comparator.comparing(i -> calculateInstanceCoef(i, ramGb, cpuCores))) .orElseThrow(IllegalArgumentException::new) .getName(); From 926f4ae64eed0efef51c1ca5630eb240044a1f4e Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 11 Sep 2019 19:43:59 +0300 Subject: [PATCH 085/194] refactoring method createTesState(PipelineRun run) with stream --- .../tesadapter/service/TaskMapper.java | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 9e7e8d1e0d..4c1b4a0a00 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -92,20 +92,17 @@ public TesTask mapToTesTask(PipelineRun run) { private TesState createTesState(PipelineRun run) { List pipelineTaskList = cloudPipelineAPIClient.loadPipelineTasks(run.getId()); - for (PipelineTask p : pipelineTaskList) { - if (p.getName().equalsIgnoreCase("Console") && pipelineTaskList.size() == 1) { - return TesState.QUEUED; - } else if (p.getName().equalsIgnoreCase("InitializeEnvironment")) { - break; - } else if ((p.getName().equalsIgnoreCase("Console") && - !p.getName().equalsIgnoreCase("InitializeEnvironment")) - && pipelineTaskList.size() > 1) { - return TesState.INITIALIZING; - } - } switch (run.getStatus()) { case RUNNING: - return TesState.RUNNING; + if (pipelineTaskList.size() == 1 && + pipelineTaskList.stream().findFirst().get().getName().equalsIgnoreCase("Console")) { + return TesState.QUEUED; + } else if (pipelineTaskList.size() > 1 && pipelineTaskList.stream() + .noneMatch(p -> p.getName().equalsIgnoreCase("InitializeEnvironment"))) { + return TesState.INITIALIZING; + } else { + return TesState.RUNNING; + } case PAUSED: return TesState.PAUSED; case SUCCESS: From e5ce0f0bb1d08bbba780530c53e2831c0727e77d Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 11 Sep 2019 19:56:25 +0300 Subject: [PATCH 086/194] refactoring method createTesTasLog. used Collections.singletonList --- .../pipeline/tesadapter/service/TaskMapper.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 60c926285d..5176600ad6 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -27,9 +27,11 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Slf4j @Service @@ -149,17 +151,12 @@ private List createListExecutor(PipelineRun run) { private List createTesTaskLog(final Long runId) { List runLogList = ListUtils.emptyIfNull(cloudPipelineAPIClient.getRunLog(runId)); - List tesExecutorLogList = new ArrayList(); - tesExecutorLogList.add(TesExecutorLog.builder() - .stdout(runLogList.stream().map(RunLog::getDate).findAny().toString() + - runLogList.stream().map(RunLog::getTaskName).iterator().next() + - runLogList.stream().map(RunLog::getLogText).iterator().next()) + List tesExecutorLogList = Collections.singletonList(TesExecutorLog.builder() + .stdout(runLogList.stream().map(i -> i.getDate()).collect(Collectors.toList()).toString()) .build()); - List tesTaskLogs = new ArrayList<>(); - tesTaskLogs.add(TesTaskLog.builder() + return Collections.singletonList(TesTaskLog.builder() .logs(tesExecutorLogList) .build()); - return ListUtils.emptyIfNull(tesTaskLogs); } @Autowired From 792b033a13097a1822dce2e1057a784ce9299ec5 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 11 Sep 2019 19:59:55 +0300 Subject: [PATCH 087/194] refactoring method createTesTaskStatus --- .../com/epam/pipeline/tesadapter/service/TaskMapper.java | 6 +++++- .../pipeline/tesadapter/service/TesTaskServiceImpl.java | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 4c1b4a0a00..3bccffd1e5 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -30,12 +30,16 @@ @Slf4j @Service public class TaskMapper { + private final String defaultInstanceType; private final Integer defaultHddSize; + @Autowired private MessageHelper messageHelper; + @Autowired private CloudPipelineAPIClient cloudPipelineAPIClient; + private static final String SEPARATOR = " "; private static final String INPUT_TYPE = "input"; private static final String OUTPUT_TYPE = "output"; @@ -95,7 +99,7 @@ private TesState createTesState(PipelineRun run) { switch (run.getStatus()) { case RUNNING: if (pipelineTaskList.size() == 1 && - pipelineTaskList.stream().findFirst().get().getName().equalsIgnoreCase("Console")) { + pipelineTaskList.get(0).getName().equalsIgnoreCase("Console")) { return TesState.QUEUED; } else if (pipelineTaskList.size() > 1 && pipelineTaskList.stream() .noneMatch(p -> p.getName().equalsIgnoreCase("InitializeEnvironment"))) { diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index d8eccfcfb5..3b608fbfe0 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -12,8 +12,8 @@ import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.vo.RunStatusVO; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.collections4.ListUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -25,16 +25,16 @@ @Slf4j @Service public class TesTaskServiceImpl implements TesTaskService { + @Value("${cloud.pipeline.service.name}") private String nameOfService; + @Value("${cloud.pipeline.doc}") private String doc; - private final CloudPipelineAPIClient cloudPipelineAPIClient; + private final CloudPipelineAPIClient cloudPipelineAPIClient; private final TaskMapper taskMapper; - private final MessageHelper messageHelper; - private final static String ID = "id"; @Autowired From 8031ef41ed3f5a4f2781363b31a7fbabee9cf1b0 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 11 Sep 2019 20:34:43 +0300 Subject: [PATCH 088/194] refactoring method createTesTaskLogs --- .../epam/pipeline/tesadapter/service/TaskMapper.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 5176600ad6..8ab97b8d2a 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -25,7 +25,6 @@ import org.springframework.stereotype.Service; import org.springframework.util.Assert; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -36,10 +35,13 @@ @Slf4j @Service public class TaskMapper { + private final String defaultInstanceType; private final Integer defaultHddSize; + @Autowired private MessageHelper messageHelper; + private CloudPipelineAPIClient cloudPipelineAPIClient; private static final String SEPARATOR = " "; private static final String INPUT_TYPE = "input"; @@ -152,7 +154,12 @@ private List createListExecutor(PipelineRun run) { private List createTesTaskLog(final Long runId) { List runLogList = ListUtils.emptyIfNull(cloudPipelineAPIClient.getRunLog(runId)); List tesExecutorLogList = Collections.singletonList(TesExecutorLog.builder() - .stdout(runLogList.stream().map(i -> i.getDate()).collect(Collectors.toList()).toString()) + .stdout(runLogList.stream() + .map(i -> i.getDate() + + i.getTaskName() + + i.getLogText()) + .collect(Collectors.toList()) + .toString()) .build()); return Collections.singletonList(TesTaskLog.builder() .logs(tesExecutorLogList) From 93114dec020335e735e085ded8cbeaaf297964d0 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 12 Sep 2019 14:38:29 +0300 Subject: [PATCH 089/194] fixed couple mistakes in TaskMapper class, added "message"s to "Exception block"s --- .../tesadapter/service/TaskMapper.java | 38 +++++++++++-------- .../tesadapter/service/TaskMapperTest.java | 2 +- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index ba0140f7ec..6d8263df70 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -50,6 +50,10 @@ public class TaskMapper { private static final String INPUT_TYPE = "input"; private static final String OUTPUT_TYPE = "output"; private static final String DEFAULT_TYPE = "string"; + private static final String TOOL = "tool"; + private static final String INSTANCE_TYPES = "instanceList"; + private static final String MIN_INSTANCE = "instance"; + private static final String REGION_ID = "id"; private static final String IMAGE = "image"; private static final String EXECUTORS = "executors"; private static final String ZONES = "zones"; @@ -122,22 +126,16 @@ public TesExecutor getExecutorFromTesExecutorsList(List tesExecutor public String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { - Double ramGb = - Optional.ofNullable(tesTask.getResources()) - .map(tesResources -> Optional.ofNullable(tesResources.getRamGb()).orElse(defaultRamGb)) - .orElse(defaultRamGb); - Long cpuCores = - Optional.ofNullable(tesTask.getResources()) - .map(tesResources -> Optional.ofNullable(tesResources.getCpuCores()).orElse(defaultCpuCore)) - .orElse(defaultCpuCore); + Double ramGb = Optional.ofNullable(tesTask.getResources()) + .map(TesResources::getRamGb).orElse(defaultRamGb); + Long cpuCores = Optional.ofNullable(tesTask.getResources()) + .map(TesResources::getCpuCores).orElse(defaultCpuCore); Long toolId = pipelineTool.getId(); Long regionId = getProperRegionIdInCloudRegionsByTesZone(Optional.ofNullable(tesTask.getResources()) - .map(tesResources -> Optional.ofNullable(tesResources.getZones()) - .orElse(Collections.singletonList(defaultRegion))) + .map(TesResources::getZones) .orElse(Collections.singletonList(defaultRegion))); Boolean spot = Optional.ofNullable(tesTask.getResources()) - .map(tesResources -> Optional.ofNullable(tesResources.getPreemptible()).orElse(defaultPreemptible)) - .orElse(defaultPreemptible); + .map(TesResources::getPreemptible).orElse(defaultPreemptible); AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = cloudPipelineAPIClient .loadAllowedInstanceAndPriceTypes(toolId, regionId, spot); Assert.notEmpty(allowedInstanceAndPriceTypes.getAllowedInstanceTypes(), messageHelper.getMessage( @@ -148,7 +146,9 @@ public String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { public Tool loadToolByTesImage(String image) { Assert.hasText(image, messageHelper.getMessage( MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, image)); - return Optional.ofNullable(cloudPipelineAPIClient.loadTool(image)).orElseThrow(IllegalArgumentException::new); + return Optional.ofNullable(cloudPipelineAPIClient.loadTool(image)).orElseThrow(() -> + new IllegalArgumentException(messageHelper + .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, TOOL))); } public Long getProperRegionIdInCloudRegionsByTesZone(List zones) { @@ -156,15 +156,21 @@ public Long getProperRegionIdInCloudRegionsByTesZone(List zones) { MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, ZONES, zones)); return Optional.ofNullable(cloudPipelineAPIClient.loadAllRegions().stream().filter( region -> region.getName().equalsIgnoreCase(zones.get(FIRST))) - .collect(Collectors.toList()).get(FIRST).getId()).orElseThrow(IllegalArgumentException::new); + .collect(Collectors.toList()).get(FIRST).getId()).orElseThrow(() -> + new IllegalArgumentException(messageHelper + .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, REGION_ID))); } private String evaluateMostProperInstanceType(AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes, Double ramGb, Long cpuCores) { return Optional.ofNullable(allowedInstanceAndPriceTypes.getAllowedInstanceTypes()) - .orElseThrow(IllegalArgumentException::new).stream() + .orElseThrow(() -> + new IllegalArgumentException(messageHelper + .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, INSTANCE_TYPES))).stream() .min(Comparator.comparing(i -> calculateInstanceCoef(i, ramGb, cpuCores))) - .orElseThrow(IllegalArgumentException::new) + .orElseThrow(() -> + new IllegalArgumentException(messageHelper + .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, MIN_INSTANCE))) .getName(); } diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index 75ee096666..aaa0ecdc81 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -129,7 +129,7 @@ void expectIllegalArgExceptionWhenRunLoadToolByTesImageWithEmptyOrNullImage() { IllegalArgumentException exception2 = assertThrows(IllegalArgumentException.class, () -> taskMapper.loadToolByTesImage(null)); assertTrue(exception2.getMessage().contains(messageHelper.getMessage( - MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, null))); + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, "null"))); } @Test From 725f91e97f2eed5bdcc609345972cda412606e2c Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 12 Sep 2019 15:17:18 +0300 Subject: [PATCH 090/194] couple clean-code, remove unnecessary imports and empty-lines --- .../java/com/epam/pipeline/tesadapter/service/TaskMapper.java | 4 +--- .../com/epam/pipeline/tesadapter/service/TaskMapperTest.java | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 656baadb32..41ecac43c6 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -5,9 +5,8 @@ import com.epam.pipeline.entity.configuration.ExecutionEnvironment; import com.epam.pipeline.entity.configuration.PipeConfValueVO; import com.epam.pipeline.entity.pipeline.PipelineRun; -import com.epam.pipeline.entity.pipeline.TaskStatus; -import com.epam.pipeline.entity.pipeline.Tool; import com.epam.pipeline.entity.pipeline.PipelineTask; +import com.epam.pipeline.entity.pipeline.Tool; import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter; import com.epam.pipeline.tesadapter.common.MessageConstants; @@ -281,6 +280,5 @@ private List createListExecutor(PipelineRun run) { .command(ListUtils.emptyIfNull(Arrays.asList(run.getActualCmd().split(SEPARATOR)))) .env(run.getEnvVars()) .build())); - } } diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index aaa0ecdc81..04b76a21e8 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -47,7 +47,6 @@ class TaskMapperTest { private static final String STUBBED_IMAGE = "cp-docker-registry.default.svc.cluster.local:31443/library/centos:latest"; private static List allowedInstanceTypes; - @Autowired private MessageHelper messageHelper; From 711c9bcf63fb455b701ed40aa119fe356186ed23 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 12 Sep 2019 18:12:52 +0300 Subject: [PATCH 091/194] added explicit scoping for TestMapperTest tests --- .../tesadapter/service/TaskMapperTest.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index 04b76a21e8..a474b191c5 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -61,13 +61,13 @@ class TaskMapperTest { private TesTask tesTask = mock(TesTask.class); @BeforeAll - static void setUpAll() { + public static void setUpAll() { allowedInstanceTypes = new ArrayList<>(); fillWithAllowedInstanceTypes(allowedInstanceTypes); } @BeforeEach - void setUp() { + public void setUp() { when(abstractCloudRegion.getName()).thenReturn(DEFAULT_REGION_NAME); when(abstractCloudRegion.getId()).thenReturn(STUBBED_REGION_ID); when(cloudPipelineAPIClient.loadAllRegions()).thenReturn(abstractCloudRegions); @@ -84,7 +84,7 @@ void setUp() { } @Test() - void expectIllegalArgExceptionWhenRunGetExecutorFromTesExecutorsList() { + public void expectIllegalArgExceptionWhenRunGetExecutorFromTesExecutorsList() { List tesExecutors = new ArrayList<>(); tesExecutors.add(new TesExecutor()); tesExecutors.add(new TesExecutor()); @@ -96,7 +96,7 @@ void expectIllegalArgExceptionWhenRunGetExecutorFromTesExecutorsList() { @ParameterizedTest @MethodSource("provideInputForGetProperInstanceTypeTest") - void getProperInstanceTypeShouldReturnProperInstanceName(String instanceTypeName, Double ramGb, Long cpuCore) { + public void getProperInstanceTypeShouldReturnProperInstanceName(String instanceTypeName, Double ramGb, Long cpuCore) { when(tesTask.getResources().getRamGb()).thenReturn(ramGb); when(tesTask.getResources().getCpuCores()).thenReturn(cpuCore); when(tesTask.getResources().getZones()).thenReturn(zones); @@ -105,7 +105,7 @@ void getProperInstanceTypeShouldReturnProperInstanceName(String instanceTypeName } @Test - void expectIllegalArgExceptionWhenRunGetProperRegionIdInCloudRegionsByTesZone() { + public void expectIllegalArgExceptionWhenRunGetProperRegionIdInCloudRegionsByTesZone() { zones.add("Second zone"); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> taskMapper.getProperRegionIdInCloudRegionsByTesZone(zones)); @@ -114,12 +114,12 @@ void expectIllegalArgExceptionWhenRunGetProperRegionIdInCloudRegionsByTesZone() } @Test - void getProperRegionIdInCloudRegionsByTesZoneShouldReturnRegionId() { + public void getProperRegionIdInCloudRegionsByTesZoneShouldReturnRegionId() { assertEquals(STUBBED_REGION_ID, taskMapper.getProperRegionIdInCloudRegionsByTesZone(zones)); } @Test - void expectIllegalArgExceptionWhenRunLoadToolByTesImageWithEmptyOrNullImage() { + public void expectIllegalArgExceptionWhenRunLoadToolByTesImageWithEmptyOrNullImage() { IllegalArgumentException exception1 = assertThrows(IllegalArgumentException.class, () -> taskMapper.loadToolByTesImage("")); assertTrue(exception1.getMessage().contains(messageHelper.getMessage( @@ -132,7 +132,7 @@ void expectIllegalArgExceptionWhenRunLoadToolByTesImageWithEmptyOrNullImage() { } @Test - void loadToolByTesImageShouldReturnToolByRequestingImage() { + public void loadToolByTesImageShouldReturnToolByRequestingImage() { assertEquals(tool, taskMapper.loadToolByTesImage(STUBBED_IMAGE)); } From 9c7baa7ba16b42ad03899e0e5549d1a7488b08c4 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Thu, 12 Sep 2019 18:16:32 +0300 Subject: [PATCH 092/194] refactoring method createTesTaskLogs. used String.format --- .../com/epam/pipeline/tesadapter/service/TaskMapper.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 8ab97b8d2a..2fc9b7f846 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -48,6 +48,8 @@ public class TaskMapper { private static final String OUTPUT_TYPE = "output"; private static final String DEFAULT_TYPE = "string"; private static final Integer FIRST = 0; + private static final String OUTPUT_LOG_STRING_FORMAT = "%s - %s - %s"; + private static final String CARRIAGE_RETURN = "\n"; public TaskMapper(@Value("${cloud.pipeline.instanceType}") String instanceType, @Value("${cloud.pipeline.hddSize}") Integer hddSize) { @@ -155,11 +157,8 @@ private List createTesTaskLog(final Long runId) { List runLogList = ListUtils.emptyIfNull(cloudPipelineAPIClient.getRunLog(runId)); List tesExecutorLogList = Collections.singletonList(TesExecutorLog.builder() .stdout(runLogList.stream() - .map(i -> i.getDate() + - i.getTaskName() + - i.getLogText()) - .collect(Collectors.toList()) - .toString()) + .map(i -> String.format(OUTPUT_LOG_STRING_FORMAT, i.getDate(), i.getTaskName(), i.getLogText())) + .collect(Collectors.joining(CARRIAGE_RETURN))) .build()); return Collections.singletonList(TesTaskLog.builder() .logs(tesExecutorLogList) From d712bc70487dd69b1b06871541f8485d1783407e Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Thu, 12 Sep 2019 21:01:52 +0300 Subject: [PATCH 093/194] implemented support cpu and ram --- .../tesadapter/service/TaskMapper.java | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 48e75cc36a..e3046136db 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -5,9 +5,8 @@ import com.epam.pipeline.entity.configuration.ExecutionEnvironment; import com.epam.pipeline.entity.configuration.PipeConfValueVO; import com.epam.pipeline.entity.pipeline.PipelineRun; -import com.epam.pipeline.entity.pipeline.RunLog; -import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.entity.pipeline.PipelineTask; +import com.epam.pipeline.entity.pipeline.RunLog; import com.epam.pipeline.entity.pipeline.Tool; import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter; @@ -38,10 +37,12 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; @Slf4j @Service public class TaskMapper { + private final Integer defaultHddSize; private final Double defaultRamGb; private final Long defaultCpuCore; @@ -72,7 +73,6 @@ public class TaskMapper { private static final Double PIB_TO_GIB = 1048576.0; private static final Double EIB_TO_GIB = 1073741824.0; - @Autowired public TaskMapper(@Value("${cloud.pipeline.hddSize}") Integer hddSize, @Value("${cloud.pipeline.ramGb}") Double defaultRamGb, @@ -279,6 +279,8 @@ private TesResources createTesResources(PipelineRun run) { return TesResources.builder() .preemptible(run.getInstance().getSpot()) .diskGb(new Double(run.getInstance().getNodeDisk())) + .ramGb(getInstanceType(run).getMemory() * convertMemoryUnitTypeToGiB(getInstanceType(run).getMemoryUnit())) + .cpuCores((long) getInstanceType(run).getVCPU()) .build(); } @@ -300,4 +302,21 @@ private List createTesTaskLog(final Long runId) { .logs(tesExecutorLogList) .build()); } + + private InstanceType getInstanceType(PipelineRun run) { + Long regionId = run.getInstance().getCloudRegionId(); + Boolean spot = run.getInstance().getSpot(); + Tool pipeLineTool = loadToolByTesImage(run.getDockerImage()); + String nodeType = run.getInstance().getNodeType(); + AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = + cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(pipeLineTool.getId(), regionId, spot); + return Stream.of(allowedInstanceAndPriceTypes) + .map(instance -> instance.getAllowedInstanceTypes()) + .flatMap(instanceType -> instanceType + .stream() + .filter(instance -> instance + .getName() + .equalsIgnoreCase(nodeType))) + .findFirst().get(); + } } From 46178e251c8542f8a08352ea35f46675477ba29a Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Fri, 13 Sep 2019 15:08:26 +0300 Subject: [PATCH 094/194] did more simply method getInstanceType using stream --- .../pipeline/tesadapter/service/TaskMapper.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index e3046136db..d79a46ac0f 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -311,12 +311,11 @@ private InstanceType getInstanceType(PipelineRun run) { AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(pipeLineTool.getId(), regionId, spot); return Stream.of(allowedInstanceAndPriceTypes) - .map(instance -> instance.getAllowedInstanceTypes()) - .flatMap(instanceType -> instanceType - .stream() - .filter(instance -> instance - .getName() - .equalsIgnoreCase(nodeType))) - .findFirst().get(); + .flatMap(instance -> instance.getAllowedInstanceTypes().stream()) + .filter(instance -> instance + .getName() + .equalsIgnoreCase(nodeType)) + .findFirst().orElseThrow(() -> new IllegalArgumentException(messageHelper + .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY))); } } From 0c94cd8d8f4addfa7f2b6fec136369da7216f004 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Fri, 13 Sep 2019 19:12:35 +0300 Subject: [PATCH 095/194] added support TaskView --- .../controller/TesAdapterController.java | 2 +- .../tesadapter/service/TaskMapper.java | 50 ++++++++++++++----- .../tesadapter/service/TesTaskService.java | 3 +- .../service/TesTaskServiceImpl.java | 5 +- 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index 5f55286bbb..52acebd974 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -50,7 +50,7 @@ ResponseEntity submitTesTask(@RequestBody TesTask body) { @GetMapping("/v1/tasks/{id}") ResponseEntity getTesTask(@PathVariable String id, @RequestParam(required = false, defaultValue = "MINIMAL") TaskView view) { - return ResponseEntity.ok().body(tesTaskService.getTesTask(id)); + return ResponseEntity.ok().body(tesTaskService.getTesTask(id, view)); } @PostMapping("/v1/tasks/{id}:cancel") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index d79a46ac0f..a3bee3c804 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -13,6 +13,7 @@ import com.epam.pipeline.tesadapter.common.MessageConstants; import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.entity.PipelineDiskMemoryTypes; +import com.epam.pipeline.tesadapter.entity.TaskView; import com.epam.pipeline.tesadapter.entity.TesExecutor; import com.epam.pipeline.tesadapter.entity.TesExecutorLog; import com.epam.pipeline.tesadapter.entity.TesInput; @@ -213,18 +214,43 @@ private Double convertMemoryUnitTypeToGiB(String memoryUnit) { return GIB_TO_GIB; } - public TesTask mapToTesTask(PipelineRun run) { - return TesTask.builder() - .id(String.valueOf(run.getId())) - .name(run.getPodId()) - .resources(createTesResources(run)) - .executors(createListExecutor(run)) - .inputs(createTesInput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) - .outputs(createTesOutput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) - .creationTime(run.getStartDate().toString()) - .logs(createTesTaskLog(run.getId())) - .state(createTesState(run)) - .build(); + public TesTask mapToTesTask(PipelineRun run, TaskView view) { + return filterTesTaskWithView(run, view); + } + + private TesTask filterTesTaskWithView(PipelineRun run, TaskView view) { + switch (view) { + case MINIMAL: + return TesTask.builder() + .id(String.valueOf(run.getId())) + .state(createTesState(run)) + .build(); + case BASIC: + return TesTask.builder() + .id(String.valueOf(run.getId())) + .name(run.getPodId()) + .resources(createTesResources(run)) + .executors(createListExecutor(run)) + .outputs(createTesOutput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) + .creationTime(run.getStartDate().toString()) + .state(createTesState(run)) + .build(); + case FULL: + return TesTask.builder() + .id(String.valueOf(run.getId())) + .name(run.getPodId()) + .resources(createTesResources(run)) + .executors(createListExecutor(run)) + .inputs(createTesInput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) + .outputs(createTesOutput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) + .creationTime(run.getStartDate().toString()) + .logs(createTesTaskLog(run.getId())) + .state(createTesState(run)) + .build(); + default: + throw new IllegalArgumentException(messageHelper.getMessage(MessageConstants. + ERROR_PARAMETER_REQUIRED)); + } } private TesState createTesState(PipelineRun run) { diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index 30c594b7b9..d567dba305 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -1,5 +1,6 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.tesadapter.entity.TaskView; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; import com.epam.pipeline.tesadapter.entity.TesServiceInfo; @@ -16,7 +17,7 @@ public interface TesTaskService { TesCancelTaskResponse cancelTesTask(String id); - TesTask getTesTask(String id); + TesTask getTesTask(String id, TaskView view); TesServiceInfo getServiceInfo(); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 3b608fbfe0..3c53bbabba 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -5,6 +5,7 @@ import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.tesadapter.common.MessageConstants; import com.epam.pipeline.tesadapter.common.MessageHelper; +import com.epam.pipeline.tesadapter.entity.TaskView; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; @@ -74,8 +75,8 @@ public TesCancelTaskResponse cancelTesTask(String id) { } @Override - public TesTask getTesTask(String id) { - return taskMapper.mapToTesTask(cloudPipelineAPIClient.loadPipelineRun(parseRunId(id))); + public TesTask getTesTask(String id, TaskView view) { + return taskMapper.mapToTesTask(cloudPipelineAPIClient.loadPipelineRun(parseRunId(id)), view); } private Long parseRunId(String id) { From c4e56c27b8aefde493f95ff70b856389372e18df Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Mon, 16 Sep 2019 13:53:37 +0300 Subject: [PATCH 096/194] added implementation for mapping tes-list-tasks method. Added corresponding updates to "common" module and tes-adapter-module --- cloud-pipeline-common/model/build.gradle | 9 + .../client/pipeline/CloudPipelineAPI.java | 5 + .../vo/PagingRunFilterExpressionVO.java | 43 +++ .../pipeline/vo/filter/AclExpression.java | 53 ++++ .../pipeline/vo/filter/AclExpressionType.java | 51 ++++ .../vo/filter/AndFilterExpression.java | 34 +++ .../pipeline/vo/filter/FilterExpression.java | 145 +++++++++ .../vo/filter/FilterExpressionType.java | 62 ++++ .../epam/pipeline/vo/filter/FilterField.java | 51 ++++ .../epam/pipeline/vo/filter/FilterFields.java | 29 ++ .../pipeline/vo/filter/FilterOperandType.java | 57 ++++ .../pipeline/vo/filter/LogicalExpression.java | 274 ++++++++++++++++++ .../vo/filter/OrFilterExpression.java | 34 +++ .../vo/filter/WrongFilterException.java | 28 ++ .../composers/AbstractFilterComposer.java | 25 ++ .../vo/filter/composers/DefaultComposer.java | 28 ++ .../AbstractFilterValueConverter.java | 41 +++ .../vo/filter/converters/DateConverter.java | 97 +++++++ .../filter/converters/DefaultConverter.java | 35 +++ .../controller/TesAdapterController.java | 5 +- .../service/CloudPipelineAPIClient.java | 6 + .../tesadapter/service/TesTaskService.java | 3 +- .../service/TesTaskServiceImpl.java | 23 +- 23 files changed, 1132 insertions(+), 6 deletions(-) create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpression.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpressionType.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AndFilterExpression.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionType.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterField.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterFields.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandType.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/OrFilterExpression.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/WrongFilterException.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/AbstractFilterComposer.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/DefaultComposer.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DateConverter.java create mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DefaultConverter.java diff --git a/cloud-pipeline-common/model/build.gradle b/cloud-pipeline-common/model/build.gradle index d5697106a4..827250a09c 100644 --- a/cloud-pipeline-common/model/build.gradle +++ b/cloud-pipeline-common/model/build.gradle @@ -61,6 +61,15 @@ dependencies { compile 'io.fabric8:kubernetes-model:1.0.67' +//AWS + compile group: "com.amazonaws", name: "aws-java-sdk-ec2", version: "1.11.377" + // https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-s3 + compile group: "com.amazonaws", name: "aws-java-sdk-s3", version: "1.11.301" + // https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-sts + compile group: "com.amazonaws", name: "aws-java-sdk-sts", version: "1.11.211" + // https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-iam + compile group: "com.amazonaws", name: "aws-java-sdk-iam", version: "1.11.211" + // Retrofit compile group: 'com.squareup.retrofit2', name: 'retrofit', version: '2.3.0' compile group: 'com.squareup.retrofit2', name: 'converter-jackson', version: '2.3.0' diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java index 022484ef17..cd537260de 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java @@ -42,10 +42,12 @@ import com.epam.pipeline.entity.region.AwsRegion; import com.epam.pipeline.entity.security.acl.AclClass; import com.epam.pipeline.entity.user.PipelineUser; +import com.epam.pipeline.rest.PagedResult; import com.epam.pipeline.rest.Result; import com.epam.pipeline.vo.EntityPermissionVO; import com.epam.pipeline.vo.EntityVO; import com.epam.pipeline.vo.FilterNodesVO; +import com.epam.pipeline.vo.PagingRunFilterExpressionVO; import com.epam.pipeline.vo.RunStatusVO; import com.epam.pipeline.vo.notification.NotificationMessageVO; import okhttp3.MultipartBody; @@ -82,6 +84,9 @@ public interface CloudPipelineAPI { @POST("run") Call> runPipeline(@Body PipelineStart runVo); + @POST("run/search") + Call>>> searchRuns(@Body PagingRunFilterExpressionVO filterVO); + @POST("run/{runId}/status") Call> updateRunStatus(@Path(RUN_ID) Long runId, @Body RunStatusVO statusUpdate); diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java new file mode 100644 index 0000000000..f8b3e566d1 --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java @@ -0,0 +1,43 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo; + +import com.epam.pipeline.entity.filter.AclSecuredFilter; +import com.epam.pipeline.vo.filter.FilterExpression; +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.util.List; + +@Getter +@Setter +@NoArgsConstructor +public class PagingRunFilterExpressionVO implements AclSecuredFilter { + + private int page; + private int pageSize; + private int timezoneOffsetInMinutes; + private FilterExpression filterExpression; + + //these filter is used for ACL filtering + @JsonIgnore + private List allowedPipelines; + @JsonIgnore + private String ownershipFilter; +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpression.java new file mode 100644 index 0000000000..f9e444ebb3 --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpression.java @@ -0,0 +1,53 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter; + +import lombok.Getter; +import lombok.Setter; + +import java.util.List; +import java.util.stream.Collectors; + +@Getter +@Setter +class AclExpression extends LogicalExpression { + private AclExpressionType aclExpressionType; + + AclExpression(AclExpressionType aclExpressionType, String value) { + super(aclExpressionType.getAclType(), value, FilterOperandType.EQUALS); + this.setAclExpressionType(aclExpressionType); + } + + AclExpression(AclExpressionType aclExpressionType, List value) { + super( + aclExpressionType.getAclType(), + String.join( + ", ", + value + .stream() + .map(Object::toString) + .collect(Collectors.toList()) + ), + FilterOperandType.EQUALS); + this.setAclExpressionType(aclExpressionType); + } + + @Override + boolean filterFieldMatches(FilterField filterField) { + return filterField.aclField() == this.getAclExpressionType(); + } +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpressionType.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpressionType.java new file mode 100644 index 0000000000..6f552eec84 --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpressionType.java @@ -0,0 +1,51 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter; + +import java.util.HashMap; +import java.util.Map; + +public enum AclExpressionType { + NONE("none"), + PIPELINE_IDS("pipeline_ids"), + OWNERSHIP("ownership"); + + private String aclType; + + private static Map operandsMap = new HashMap<>(); + static { + operandsMap.put(NONE.aclType, NONE); + operandsMap.put(PIPELINE_IDS.aclType, PIPELINE_IDS); + operandsMap.put(OWNERSHIP.aclType, OWNERSHIP); + } + + AclExpressionType(String type) { + this.aclType = type; + } + + public static AclExpressionType getByString(String type) { + if (type == null) { + return null; + } + return operandsMap.get(type); + } + + + public String getAclType() { + return this.aclType; + } +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AndFilterExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AndFilterExpression.java new file mode 100644 index 0000000000..ea63f0314d --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AndFilterExpression.java @@ -0,0 +1,34 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter; + +public class AndFilterExpression extends FilterExpression { + + AndFilterExpression() { + super(); + } + + @Override + public String toSQLStatement() throws WrongFilterException { + if (this.getExpressions() != null && this.getExpressions().size() == 2) { + return String.format("(%s AND %s)", + this.getExpressions().get(0).toSQLStatement(), + this.getExpressions().get(1).toSQLStatement()); + } + throw new WrongFilterException(); + } +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java new file mode 100644 index 0000000000..e43d23b37e --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java @@ -0,0 +1,145 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter; + +import com.amazonaws.util.StringUtils; +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +@Getter +@Setter +@NoArgsConstructor +public class FilterExpression { + + private String field; + private String value; + private String operand; + @JsonIgnore + private FilterOperandType operandType; + @JsonIgnore + private FilterField associatedFilterField; + + private List expressions; + private FilterExpressionType filterExpressionType; + + public String toSQLStatement() throws WrongFilterException { + throw new WrongFilterException(); + } + + public FilterExpression preProcessExpression(Class context, + MapSqlParameterSource parameterSource, + Map parametersPlaceholders, + Map params) + throws WrongFilterException { + if (this.expressions != null) { + List childExpressions = new ArrayList<>(); + for (FilterExpression childExpression : this.expressions) { + childExpressions.add( + childExpression.preProcessExpression( + context, + parameterSource, + parametersPlaceholders, + params) + ); + } + this.expressions = childExpressions; + } + return this; + } + + static FilterExpression generate(FilterExpression rootExpression, + List allowedPipelines, + String ownership) + throws WrongFilterException { + boolean containsOwnershipFilter = !StringUtils.isNullOrEmpty(ownership); + boolean containsPipelineIdsFilter = allowedPipelines != null && allowedPipelines.size() > 0; + if (containsPipelineIdsFilter && containsOwnershipFilter) { + AndFilterExpression mainExpression = new AndFilterExpression(); + OrFilterExpression aclExpression = new OrFilterExpression(); + aclExpression.setExpressions(Arrays.asList( + new AclExpression(AclExpressionType.PIPELINE_IDS, allowedPipelines), + new AclExpression(AclExpressionType.OWNERSHIP, ownership) + )); + mainExpression.setExpressions(Arrays.asList( + FilterExpression.prepare(rootExpression), + aclExpression + )); + return mainExpression; + } else if (containsOwnershipFilter) { + AndFilterExpression mainExpression = new AndFilterExpression(); + mainExpression.setExpressions(Arrays.asList( + FilterExpression.prepare(rootExpression), + new AclExpression(AclExpressionType.OWNERSHIP, ownership) + )); + return mainExpression; + } else if (containsPipelineIdsFilter) { + AndFilterExpression mainExpression = new AndFilterExpression(); + mainExpression.setExpressions(Arrays.asList( + FilterExpression.prepare(rootExpression), + new AclExpression(AclExpressionType.PIPELINE_IDS, allowedPipelines) + )); + return mainExpression; + } else { + return FilterExpression.prepare(rootExpression); + } + } + + private static FilterExpression prepare(FilterExpression expression) throws WrongFilterException { + FilterExpression result = null; + if (expression.getFilterExpressionType() == null) { + throw new WrongFilterException("Unknown expression type"); + } + switch (expression.filterExpressionType) { + case LOGICAL: + if (FilterOperandType.getByString(expression.getOperand()) == null) { + throw new WrongFilterException( + String.format("Unknown or incorrect operand: '%s'", + expression.getOperand()) + ); + } + result = new LogicalExpression( + expression.getField(), + expression.getValue(), + FilterOperandType.getByString(expression.getOperand())); + break; + case OR: + result = new OrFilterExpression(); + break; + case AND: + result = new AndFilterExpression(); + break; + default: + break; + } + if (expression.getExpressions() != null && result != null) { + result.setExpressions(new ArrayList<>()); + for (FilterExpression child : expression.getExpressions()) { + result.getExpressions().add(FilterExpression.prepare(child)); + } + } + return result; + } + +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionType.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionType.java new file mode 100644 index 0000000000..d2dc44a67e --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionType.java @@ -0,0 +1,62 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter; + +import java.util.HashMap; +import java.util.Map; + +public enum FilterExpressionType { + LOGICAL(0), + AND(1), + OR(2); + + private long id; + private static Map idMap = new HashMap<>(); + static { + idMap.put(LOGICAL.id, LOGICAL); + idMap.put(AND.id, AND); + idMap.put(OR.id, OR); + } + private static Map namesMap = new HashMap<>(); + static { + namesMap.put(LOGICAL.name(), LOGICAL); + namesMap.put(AND.name(), AND); + namesMap.put(OR.name(), OR); + } + + FilterExpressionType(long id) { + this.id = id; + } + + public static FilterExpressionType getById(Long id) { + if (id == null) { + return null; + } + return idMap.get(id); + } + + public static FilterExpressionType getByName(String name) { + if (name == null) { + return null; + } + return namesMap.get(name); + } + + public Long getId() { + return this.id; + } +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterField.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterField.java new file mode 100644 index 0000000000..2d7a6da00c --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterField.java @@ -0,0 +1,51 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter; + +import com.epam.pipeline.vo.filter.composers.AbstractFilterComposer; +import com.epam.pipeline.vo.filter.composers.DefaultComposer; +import com.epam.pipeline.vo.filter.converters.AbstractFilterValueConverter; +import com.epam.pipeline.vo.filter.converters.DefaultConverter; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Repeatable; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.METHOD, ElementType.FIELD}) +@Retention(RetentionPolicy.RUNTIME) +@Repeatable(FilterFields.class) +public @interface FilterField { + String displayName() default ""; + String databaseFieldName() default ""; + String databaseTableAlias() default ""; + String description() default ""; + AclExpressionType aclField() default AclExpressionType.NONE; + boolean isRegex() default false; + boolean multiplePlaceholders() default false; + FilterOperandType[] supportedOperands() default { + FilterOperandType.LESS, + FilterOperandType.LESS_OR_EQUALS, + FilterOperandType.EQUALS, + FilterOperandType.NOT_EQUALS, + FilterOperandType.MORE_OR_EQUALS, + FilterOperandType.MORE + }; + Class converter() default DefaultConverter.class; + Class composer() default DefaultComposer.class; +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterFields.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterFields.java new file mode 100644 index 0000000000..44ce0574a0 --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterFields.java @@ -0,0 +1,29 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter; + + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.METHOD, ElementType.FIELD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface FilterFields { + FilterField[] value(); +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandType.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandType.java new file mode 100644 index 0000000000..b53e3b4d72 --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandType.java @@ -0,0 +1,57 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter; + +import java.util.HashMap; +import java.util.Map; + +public enum FilterOperandType { + LESS("<"), + LESS_OR_EQUALS("<="), + EQUALS("="), + NOT_EQUALS("!="), + MORE_OR_EQUALS(">="), + MORE(">"); + + private String operand; + + private static Map operandsMap = new HashMap<>(); + static { + operandsMap.put(LESS.operand, LESS); + operandsMap.put(LESS_OR_EQUALS.operand, LESS_OR_EQUALS); + operandsMap.put(EQUALS.operand, EQUALS); + operandsMap.put(NOT_EQUALS.operand, NOT_EQUALS); + operandsMap.put(MORE_OR_EQUALS.operand, MORE_OR_EQUALS); + operandsMap.put(MORE.operand, MORE); + } + + FilterOperandType(String operand) { + this.operand = operand; + } + + public static FilterOperandType getByString(String operand) { + if (operand == null) { + return null; + } + return operandsMap.get(operand); + } + + + public String getOperand() { + return this.operand; + } +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java new file mode 100644 index 0000000000..064a1b97c8 --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java @@ -0,0 +1,274 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter; + +import com.amazonaws.util.StringUtils; +import com.epam.pipeline.vo.filter.composers.AbstractFilterComposer; +import com.epam.pipeline.vo.filter.converters.AbstractFilterValueConverter; +import com.epam.pipeline.vo.filter.converters.DateConverter; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +@Data +@Slf4j +@EqualsAndHashCode(callSuper = false) +public class LogicalExpression extends FilterExpression { + + private String databaseFieldName; + private String databaseValuePlaceholder; + private List databaseValuePlaceholders; + private boolean multiplePlaceholders; + + private AbstractFilterComposer composer; + + private LogicalExpression() { + super(); + } + + LogicalExpression(String field, String value, FilterOperandType operand) { + this(); + this.setField(field); + this.setValue(value); + this.setOperandType(operand); + } + + public String generateSQLStatement() throws WrongFilterException { + if (StringUtils.isNullOrEmpty(this.databaseFieldName) + || StringUtils.isNullOrEmpty(this.databaseValuePlaceholder)) { + throw new WrongFilterException(); + } + return String.format("(%s %s :%s)", + this.databaseFieldName, + this.getOperandType().getOperand(), + this.databaseValuePlaceholder); + } + + @Override + public String toSQLStatement() throws WrongFilterException { + if (this.composer == null) { + throw new WrongFilterException(); + } + return this.composer.generateExpression(this); + } + + private String getValuePlaceholder(Field field, + Map parametersPlaceholders) { + String placeholder = field.getName(); + if (!parametersPlaceholders.containsKey(placeholder)) { + parametersPlaceholders.put(placeholder, 0L); + } else { + Long placeholderCount = parametersPlaceholders.get(placeholder) + 1; + parametersPlaceholders.replace(placeholder, placeholderCount); + placeholder = String.format("%s_%d", placeholder, placeholderCount); + } + return placeholder; + } + + private void associateFilterField(Field field, + FilterField filterField, + MapSqlParameterSource parameterSource, + Map parametersPlaceholders, + Map params) + throws WrongFilterException { + if (!Arrays.asList(filterField.supportedOperands()).contains(this.getOperandType())) { + final String supportedOperands = String.join( + ", ", + Arrays.stream(filterField.supportedOperands()) + .map(operand -> String.format("'%s'", operand.getOperand())) + .collect(Collectors.toList()) + ); + throw new WrongFilterException( + String.format( + "Operand '%s' is not supported for field '%s'. Supported operands are: %s", + this.getOperandType().getOperand(), + this.getField(), + supportedOperands + ) + ); + } + String dbFieldName = filterField.databaseFieldName(); + if (StringUtils.isNullOrEmpty(dbFieldName)) { + dbFieldName = field.getName(); + } + if (StringUtils.isNullOrEmpty(filterField.databaseTableAlias())) { + this.databaseFieldName = dbFieldName; + } else { + this.databaseFieldName = String.format("%s.%s", + filterField.databaseTableAlias(), + dbFieldName); + } + + Object value; + AbstractFilterValueConverter converter = null; + try { + Constructor constructor = filterField.converter() + .getConstructor(); + converter = constructor.newInstance(); + } catch (NoSuchMethodException | + IllegalAccessException | + InvocationTargetException | + InstantiationException e) { + log.error(e.getMessage(), e); + } + if (converter != null) { + value = converter.convert( + this.getField(), + this.getValue(), + this.getOperandType(), + params); + } else { + value = this.getValue(); + } + + this.multiplePlaceholders = filterField.multiplePlaceholders() && value instanceof Collection; + if (this.multiplePlaceholders) { + this.databaseValuePlaceholders = new ArrayList<>(); + for (Object valueItem : (Collection) value) { + String placeholder = this.getValuePlaceholder(field, parametersPlaceholders); + this.databaseValuePlaceholders.add(placeholder); + parameterSource.addValue(placeholder, valueItem); + } + } else { + this.databaseValuePlaceholder = this.getValuePlaceholder(field, parametersPlaceholders); + parameterSource.addValue(this.databaseValuePlaceholder, value); + } + + try { + Constructor constructor = filterField.composer() + .getConstructor(); + this.composer = constructor.newInstance(); + } catch (NoSuchMethodException | + IllegalAccessException | + InvocationTargetException | + InstantiationException e) { + log.error(e.getMessage(), e); + } + this.setAssociatedFilterField(filterField); + } + + private FilterExpression preProcessField(Field field, + FilterField filterField, + Class context, + MapSqlParameterSource parameterSource, + Map parametersPlaceholders, + Map params) + throws WrongFilterException { + if (DateConverter.class.isAssignableFrom(filterField.converter()) && + ( + this.getOperandType() == FilterOperandType.EQUALS || + this.getOperandType() == FilterOperandType.NOT_EQUALS + )) { + if (this.getOperandType() == FilterOperandType.EQUALS) { + AndFilterExpression andFilterExpression = new AndFilterExpression(); + List expressionList = new ArrayList<>(); + LogicalExpression leftExpression = new LogicalExpression( + this.getField(), + this.getValue(), + FilterOperandType.MORE_OR_EQUALS); + LogicalExpression rightExpression = new LogicalExpression( + this.getField(), + this.getValue(), + FilterOperandType.LESS_OR_EQUALS); + expressionList.add(leftExpression); + expressionList.add(rightExpression); + andFilterExpression.setExpressions(expressionList); + return andFilterExpression.preProcessExpression( + context, + parameterSource, + parametersPlaceholders, + params); + } else { + OrFilterExpression orFilterExpression = new OrFilterExpression(); + List expressionList = new ArrayList<>(); + LogicalExpression leftExpression = new LogicalExpression( + this.getField(), + this.getValue(), + FilterOperandType.MORE); + LogicalExpression rightExpression = new LogicalExpression( + this.getField(), + this.getValue(), + FilterOperandType.LESS); + expressionList.add(leftExpression); + expressionList.add(rightExpression); + orFilterExpression.setExpressions(expressionList); + return orFilterExpression.preProcessExpression( + context, + parameterSource, + parametersPlaceholders, + params); + } + } else { + this.associateFilterField(field, filterField, parameterSource, parametersPlaceholders, params); + return this; + } + } + + boolean filterFieldMatches(FilterField filterField) { + return Pattern.matches(filterField.displayName(), this.getField()) || + filterField.displayName().equalsIgnoreCase(this.getField()); + } + + @Override + public FilterExpression preProcessExpression(Class context, + MapSqlParameterSource parameterSource, + Map parametersPlaceholders, + Map params) + throws WrongFilterException { + Field[] fields = context.getFields(); + for (Field field : fields) { + FilterFields filterFields = field.getAnnotation(FilterFields.class); + if (filterFields != null) { + for (FilterField filterField : filterFields.value()) { + if (this.filterFieldMatches(filterField)) { + return this.preProcessField( + field, + filterField, + context, + parameterSource, + parametersPlaceholders, + params); + } + } + } else { + FilterField filterField = field.getAnnotation(FilterField.class); + if (filterField != null && this.filterFieldMatches(filterField)) { + return this.preProcessField( + field, + filterField, + context, + parameterSource, + parametersPlaceholders, + params); + } + } + } + throw new WrongFilterException(String.format("Unknown field: %s", this.getField())); + } +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/OrFilterExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/OrFilterExpression.java new file mode 100644 index 0000000000..7db4fc597f --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/OrFilterExpression.java @@ -0,0 +1,34 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter; + +public class OrFilterExpression extends FilterExpression { + + OrFilterExpression() { + super(); + } + + @Override + public String toSQLStatement() throws WrongFilterException { + if (this.getExpressions() != null && this.getExpressions().size() == 2) { + return String.format("(%s OR %s)", + this.getExpressions().get(0).toSQLStatement(), + this.getExpressions().get(1).toSQLStatement()); + } + throw new WrongFilterException(); + } +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/WrongFilterException.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/WrongFilterException.java new file mode 100644 index 0000000000..34c5eb746d --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/WrongFilterException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter; + +public class WrongFilterException extends Exception { + + public WrongFilterException() { + + } + + public WrongFilterException(String message) { + super(message); + } +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/AbstractFilterComposer.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/AbstractFilterComposer.java new file mode 100644 index 0000000000..54579eaec4 --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/AbstractFilterComposer.java @@ -0,0 +1,25 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter.composers; + + +import com.epam.pipeline.vo.filter.LogicalExpression; +import com.epam.pipeline.vo.filter.WrongFilterException; + +public abstract class AbstractFilterComposer { + public abstract String generateExpression(LogicalExpression expression) throws WrongFilterException; +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/DefaultComposer.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/DefaultComposer.java new file mode 100644 index 0000000000..7492f17e61 --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/DefaultComposer.java @@ -0,0 +1,28 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter.composers; + + +import com.epam.pipeline.vo.filter.LogicalExpression; +import com.epam.pipeline.vo.filter.WrongFilterException; + +public class DefaultComposer extends AbstractFilterComposer { + @Override + public String generateExpression(LogicalExpression expression) throws WrongFilterException { + return expression.generateSQLStatement(); + } +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java new file mode 100644 index 0000000000..10ba3cea93 --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java @@ -0,0 +1,41 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter.converters; + + +import com.amazonaws.util.StringUtils; +import com.epam.pipeline.vo.filter.FilterOperandType; +import com.epam.pipeline.vo.filter.WrongFilterException; + +import java.util.Map; + +public abstract class AbstractFilterValueConverter { + + String clearQuotes(String value) { + if (!StringUtils.isNullOrEmpty(value) && ((value.startsWith("\"") && value.endsWith("\"")) || + (value.startsWith("'") && value.endsWith("'")))) { + return value.substring(1, value.length() - 1); + } + return value; + } + + public abstract Object convert(String field, + String value, + FilterOperandType operandType, + Map params) + throws WrongFilterException; +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DateConverter.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DateConverter.java new file mode 100644 index 0000000000..4b21b48ab5 --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DateConverter.java @@ -0,0 +1,97 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter.converters; + + +import com.amazonaws.util.StringUtils; +import com.epam.pipeline.vo.filter.FilterOperandType; +import com.epam.pipeline.vo.filter.WrongFilterException; +import lombok.extern.slf4j.Slf4j; + +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.Map; +import java.util.TimeZone; + +@Slf4j +public class DateConverter extends AbstractFilterValueConverter { + + public static final String TIMEZONE_OFFSET_PARAMETER = "Timezone offset"; + + private static final String DATE_FORMAT = "yyyy-MM-dd"; + + private DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); + private Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + + private static final int END_OF_DAY_HOURS = 23; + private static final int END_OF_DAY_MINUTES = 59; + private static final int END_OF_DAY_SECONDS = 59; + private static final int END_OF_DAY_MILLISECONDS = 999; + + @Override + public Object convert(String field, + String value, + FilterOperandType operandType, + Map params) + throws WrongFilterException { + Date result = null; + String dateValue = this.clearQuotes(value); + int timezoneOffset = 0; + if (params != null && params.containsKey(TIMEZONE_OFFSET_PARAMETER)) { + timezoneOffset = Integer.parseInt(params.get(TIMEZONE_OFFSET_PARAMETER).toString()); + } + if (!StringUtils.isNullOrEmpty(dateValue)) { + try { + result = dateFormat.parse(dateValue); + if (!dateValue.equals(dateFormat.format(result))) { + result = null; + } else { + this.calendar.setTime(result); + switch (operandType) { + case LESS_OR_EQUALS: + case MORE: + this.calendar.set(Calendar.HOUR, END_OF_DAY_HOURS); + this.calendar.set(Calendar.MINUTE, END_OF_DAY_MINUTES); + this.calendar.set(Calendar.SECOND, END_OF_DAY_SECONDS); + this.calendar.set(Calendar.MILLISECOND, END_OF_DAY_MILLISECONDS); + break; + default: + this.calendar.set(Calendar.HOUR, 0); + this.calendar.set(Calendar.MINUTE, 0); + this.calendar.set(Calendar.SECOND, 0); + this.calendar.set(Calendar.MILLISECOND, 0); + break; + } + this.calendar.add(Calendar.MINUTE, -timezoneOffset); + result = this.calendar.getTime(); + } + } catch (ParseException e) { + log.error(e.getMessage(), e); + } + if (result != null) { + return result; + } + } + throw new WrongFilterException(String.format( + "Wrong date format (%s). Allowed format is: %s", + value, + DATE_FORMAT)); + } +} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DefaultConverter.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DefaultConverter.java new file mode 100644 index 0000000000..3fc9e2771f --- /dev/null +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DefaultConverter.java @@ -0,0 +1,35 @@ +/* + * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.vo.filter.converters; + + +import com.epam.pipeline.vo.filter.FilterOperandType; +import com.epam.pipeline.vo.filter.WrongFilterException; + +import java.util.Map; + +public class DefaultConverter extends AbstractFilterValueConverter { + + @Override + public Object convert(String field, + String value, + FilterOperandType operandType, + Map params) + throws WrongFilterException { + return this.clearQuotes(value); + } +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index 5f55286bbb..160df6128c 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -36,10 +36,11 @@ public ResponseEntity serviceInfo() { @GetMapping("/v1/tasks") public ResponseEntity listTesTasks( @RequestParam(name = "name_prefix", required = false) String namePrefix, - @RequestParam(name = "page_size", required = false) long pageSize, + @RequestParam(name = "page_size", required = false) Long pageSize, @RequestParam(name = "page_token", required = false) String pageToken, @RequestParam(name = "view", required = false, defaultValue = "MINIMAL") TaskView view) { - return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.listTesTask()); + return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.listTesTask(namePrefix, pageSize, pageToken, + view)); } @PostMapping("/v1/tasks") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index d4eef5fca8..8f2ff4914f 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -10,7 +10,9 @@ import com.epam.pipeline.entity.pipeline.RunLog; import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.entity.region.AbstractCloudRegion; +import com.epam.pipeline.rest.PagedResult; import com.epam.pipeline.utils.QueryUtils; +import com.epam.pipeline.vo.PagingRunFilterExpressionVO; import com.epam.pipeline.vo.RunStatusVO; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -65,4 +67,8 @@ public List getRunLog(final Long runId){ public List loadPipelineTasks(final Long id){ return QueryUtils.execute(cloudPipelineAPI.loadPipelineTasks(id)); } + + public PagedResult> searchRuns(PagingRunFilterExpressionVO filterVO) { + return QueryUtils.execute(cloudPipelineAPI.searchRuns(filterVO)); + } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index 30c594b7b9..828fdb73c5 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -1,5 +1,6 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.tesadapter.entity.TaskView; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; import com.epam.pipeline.tesadapter.entity.TesServiceInfo; @@ -10,7 +11,7 @@ public interface TesTaskService { TesCreateTaskResponse submitTesTask(TesTask body); - TesListTasksResponse listTesTask(); + TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String pageToken, TaskView view); void stub(); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 3b608fbfe0..a500cf1d4b 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -5,21 +5,24 @@ import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.tesadapter.common.MessageConstants; import com.epam.pipeline.tesadapter.common.MessageHelper; +import com.epam.pipeline.tesadapter.entity.TaskView; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; import com.epam.pipeline.tesadapter.entity.TesServiceInfo; import com.epam.pipeline.tesadapter.entity.TesTask; +import com.epam.pipeline.vo.PagingRunFilterExpressionVO; import com.epam.pipeline.vo.RunStatusVO; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; @Slf4j @@ -36,6 +39,8 @@ public class TesTaskServiceImpl implements TesTaskService { private final TaskMapper taskMapper; private final MessageHelper messageHelper; private final static String ID = "id"; + private static final String defaultPageToken = "1"; + private static final Long defaultPageSize = 256L; @Autowired public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient, TaskMapper taskMapper, @@ -56,8 +61,20 @@ public TesCreateTaskResponse submitTesTask(TesTask body) { } @Override - public TesListTasksResponse listTesTask() { - return new TesListTasksResponse(); + public TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String pageToken, TaskView view) { + //TODO implement "namePrefix" and "view" in "filter" + TesListTasksResponse tesListTasksResponse = new TesListTasksResponse(); + PagingRunFilterExpressionVO filterExpressionVO = new PagingRunFilterExpressionVO(); + + String optionalPageToken = Optional.ofNullable(pageToken) + .orElse(defaultPageToken); + filterExpressionVO.setPage(Integer.parseInt(optionalPageToken)); + Long optionalPageSize = Optional.ofNullable(pageSize).orElse(defaultPageSize); + filterExpressionVO.setPageSize(optionalPageSize.intValue()); + List pipelineRunList = cloudPipelineAPIClient.searchRuns(filterExpressionVO).getElements(); + tesListTasksResponse.setTasks(pipelineRunList.stream().map(taskMapper::mapToTesTask) + .collect(Collectors.toList())); + return tesListTasksResponse; } @Override From 8731d4c52a1005d86154d7066ad810559ef942da Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Mon, 16 Sep 2019 14:00:43 +0300 Subject: [PATCH 097/194] added couple changes for listTesTask Test in Mvc and Unit test classes --- .../app/TesAdapterControllerTest.java | 14 +++++++--- .../service/TesTaskServiceImplTest.java | 28 +++++++++++++++---- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 3106db2364..ba39db53ff 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -2,6 +2,7 @@ import com.epam.pipeline.tesadapter.controller.TesAdapterController; +import com.epam.pipeline.tesadapter.entity.TaskView; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; @@ -23,7 +24,6 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; - import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -32,7 +32,10 @@ @SuppressWarnings({"unused", "PMD.TooManyStaticImports"}) public class TesAdapterControllerTest { private static final String STUBBED_TASK_ID = "5"; - private static final Long STUBBED_PAGE_SIZE = 55L; + private static final String PAGE_TOKEN = "1"; + private static final String NAME_PREFIX = "pipeline"; + private static final Long PAGE_SIZE = 20L; + private static final TaskView DEFAULT_VIEW = TaskView.BASIC; private static final String STUBBED_SUBMIT_JSON_REQUEST = "{}"; private static final String STUBBED_SUBMIT_JSON_RESPONSE = "{\"id\":\"5\"}"; @@ -64,8 +67,11 @@ void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { @Test void listTesTaskWhenRequestingReturnTesListTasksResponse() throws Exception { - when(tesTaskService.listTesTask()).thenReturn(new TesListTasksResponse()); - this.mockMvc.perform(get("/v1/tasks?page_size={size}", STUBBED_PAGE_SIZE)) + when(tesTaskService.listTesTask(NAME_PREFIX, PAGE_SIZE, PAGE_TOKEN, DEFAULT_VIEW)) + .thenReturn(new TesListTasksResponse()); + this.mockMvc.perform(get("/v1/tasks?name_prefix={name_prefix}?page_size={page_size}" + + "?page_token={page_token}?view={view}", + NAME_PREFIX, PAGE_SIZE, PAGE_TOKEN, DEFAULT_VIEW)) .andDo(print()).andExpect(status().isOk()); } } diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java index 3bf61e6b2f..5c5ce9b74f 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java @@ -1,23 +1,41 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.rest.PagedResult; import com.epam.pipeline.tesadapter.common.MessageHelper; +import com.epam.pipeline.tesadapter.entity.TaskView; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.List; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; class TesTaskServiceImplTest { - private CloudPipelineAPIClient cloudPipelineAPIClient; private TesTaskServiceImpl tesTaskService; + private static final String PAGE_TOKEN = "1"; + private static final String NAME_PREFIX = "pipeline"; + private static final Long PAGE_SIZE = 20L; + private static final Integer NUMBER_OF_RUNS = 20; + private static final TaskView DEFAULT_VIEW = TaskView.BASIC; + + private List pipelineRunList = new ArrayList<>(); + private PagedResult> pagedPipelineRunList = new PagedResult<>(pipelineRunList, NUMBER_OF_RUNS); + private CloudPipelineAPIClient cloudPipelineAPIClient = mock(CloudPipelineAPIClient.class); @BeforeEach public void setUp() { - cloudPipelineAPIClient = Mockito.mock(CloudPipelineAPIClient.class); - tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient, Mockito.mock(TaskMapper.class), Mockito.mock(MessageHelper.class)); + when(cloudPipelineAPIClient.searchRuns(any())).thenReturn(pagedPipelineRunList); + tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient, mock(TaskMapper.class), + mock(MessageHelper.class)); } @Test void listTesTask() { - Assertions.assertNotNull(tesTaskService.listTesTask()); + Assertions.assertNotNull(tesTaskService.listTesTask(NAME_PREFIX, PAGE_SIZE, PAGE_TOKEN, DEFAULT_VIEW)); } } \ No newline at end of file From b520efcf3253c055c308fb6d0ef6156c0de68783 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Mon, 16 Sep 2019 14:48:53 +0300 Subject: [PATCH 098/194] added implementation in listTesTask method for "namePrefix" and "view" params --- .../pipeline/tesadapter/entity/TaskView.java | 4 ++- .../service/TesTaskServiceImpl.java | 25 +++++++++++-------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskView.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskView.java index ace90a02cd..432d737dce 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskView.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskView.java @@ -1,8 +1,10 @@ package com.epam.pipeline.tesadapter.entity; +import lombok.ToString; + +@ToString public enum TaskView { MINIMAL, BASIC, FULL; - } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index a500cf1d4b..cd6b851414 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -28,7 +28,6 @@ @Slf4j @Service public class TesTaskServiceImpl implements TesTaskService { - @Value("${cloud.pipeline.service.name}") private String nameOfService; @@ -38,9 +37,13 @@ public class TesTaskServiceImpl implements TesTaskService { private final CloudPipelineAPIClient cloudPipelineAPIClient; private final TaskMapper taskMapper; private final MessageHelper messageHelper; - private final static String ID = "id"; - private static final String defaultPageToken = "1"; - private static final Long defaultPageSize = 256L; + + private static final String ID = "id"; + private static final String NAME_PREFIX = "namePrefix"; + private static final String VIEW = "view"; + private static final String DEFAULT_PAGE_TOKEN = "1"; + private static final Long DEFAULT_PAGE_SIZE = 256L; + private static final TaskView DEFAULT_TASK_VIEW = TaskView.BASIC; @Autowired public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient, TaskMapper taskMapper, @@ -62,15 +65,17 @@ public TesCreateTaskResponse submitTesTask(TesTask body) { @Override public TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String pageToken, TaskView view) { - //TODO implement "namePrefix" and "view" in "filter" TesListTasksResponse tesListTasksResponse = new TesListTasksResponse(); PagingRunFilterExpressionVO filterExpressionVO = new PagingRunFilterExpressionVO(); - String optionalPageToken = Optional.ofNullable(pageToken) - .orElse(defaultPageToken); - filterExpressionVO.setPage(Integer.parseInt(optionalPageToken)); - Long optionalPageSize = Optional.ofNullable(pageSize).orElse(defaultPageSize); - filterExpressionVO.setPageSize(optionalPageSize.intValue()); + filterExpressionVO.setPage(Integer.parseInt(Optional.ofNullable(pageToken).orElse(DEFAULT_PAGE_TOKEN))); + filterExpressionVO.setPageSize(Optional.ofNullable(pageSize).orElse(DEFAULT_PAGE_SIZE).intValue()); + filterExpressionVO.getFilterExpression().setField(VIEW); + filterExpressionVO.getFilterExpression().setValue(Optional.ofNullable(view).orElse(DEFAULT_TASK_VIEW).toString()); + Optional.ofNullable(namePrefix).ifPresent((name) -> { + filterExpressionVO.getFilterExpression().setField(NAME_PREFIX); + filterExpressionVO.getFilterExpression().setValue(name); + }); List pipelineRunList = cloudPipelineAPIClient.searchRuns(filterExpressionVO).getElements(); tesListTasksResponse.setTasks(pipelineRunList.stream().map(taskMapper::mapToTesTask) .collect(Collectors.toList())); From c39191642d36e401c8603ea66eafc630af6a88e1 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Mon, 16 Sep 2019 17:06:47 +0300 Subject: [PATCH 099/194] refactored listTesTask method, added FilterExpression config. Changed StringUtils import from AWS to Apache package - changed corresponding files. --- cloud-pipeline-common/model/build.gradle | 9 -------- .../pipeline/vo/filter/FilterExpression.java | 12 +++++----- .../pipeline/vo/filter/LogicalExpression.java | 10 ++++---- .../AbstractFilterValueConverter.java | 4 ++-- .../vo/filter/converters/DateConverter.java | 4 ++-- .../service/TesTaskServiceImpl.java | 23 +++++++++++-------- 6 files changed, 28 insertions(+), 34 deletions(-) diff --git a/cloud-pipeline-common/model/build.gradle b/cloud-pipeline-common/model/build.gradle index 827250a09c..d5697106a4 100644 --- a/cloud-pipeline-common/model/build.gradle +++ b/cloud-pipeline-common/model/build.gradle @@ -61,15 +61,6 @@ dependencies { compile 'io.fabric8:kubernetes-model:1.0.67' -//AWS - compile group: "com.amazonaws", name: "aws-java-sdk-ec2", version: "1.11.377" - // https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-s3 - compile group: "com.amazonaws", name: "aws-java-sdk-s3", version: "1.11.301" - // https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-sts - compile group: "com.amazonaws", name: "aws-java-sdk-sts", version: "1.11.211" - // https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-iam - compile group: "com.amazonaws", name: "aws-java-sdk-iam", version: "1.11.211" - // Retrofit compile group: 'com.squareup.retrofit2', name: 'retrofit', version: '2.3.0' compile group: 'com.squareup.retrofit2', name: 'converter-jackson', version: '2.3.0' diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java index e43d23b37e..74968d14ea 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java @@ -16,11 +16,11 @@ package com.epam.pipeline.vo.filter; -import com.amazonaws.util.StringUtils; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; +import org.apache.commons.lang3.StringUtils; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import java.util.ArrayList; @@ -72,8 +72,8 @@ public FilterExpression preProcessExpression(Class context, static FilterExpression generate(FilterExpression rootExpression, List allowedPipelines, String ownership) - throws WrongFilterException { - boolean containsOwnershipFilter = !StringUtils.isNullOrEmpty(ownership); + throws WrongFilterException { + boolean containsOwnershipFilter = StringUtils.isNotEmpty(ownership); boolean containsPipelineIdsFilter = allowedPipelines != null && allowedPipelines.size() > 0; if (containsPipelineIdsFilter && containsOwnershipFilter) { AndFilterExpression mainExpression = new AndFilterExpression(); @@ -120,9 +120,9 @@ private static FilterExpression prepare(FilterExpression expression) throws Wron ); } result = new LogicalExpression( - expression.getField(), - expression.getValue(), - FilterOperandType.getByString(expression.getOperand())); + expression.getField(), + expression.getValue(), + FilterOperandType.getByString(expression.getOperand())); break; case OR: result = new OrFilterExpression(); diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java index 064a1b97c8..f692ac314c 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java @@ -16,13 +16,13 @@ package com.epam.pipeline.vo.filter; -import com.amazonaws.util.StringUtils; import com.epam.pipeline.vo.filter.composers.AbstractFilterComposer; import com.epam.pipeline.vo.filter.converters.AbstractFilterValueConverter; import com.epam.pipeline.vo.filter.converters.DateConverter; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import java.lang.reflect.Constructor; @@ -60,8 +60,8 @@ private LogicalExpression() { } public String generateSQLStatement() throws WrongFilterException { - if (StringUtils.isNullOrEmpty(this.databaseFieldName) - || StringUtils.isNullOrEmpty(this.databaseValuePlaceholder)) { + if (StringUtils.isEmpty(this.databaseFieldName) + || StringUtils.isEmpty(this.databaseValuePlaceholder)) { throw new WrongFilterException(); } return String.format("(%s %s :%s)", @@ -114,10 +114,10 @@ private void associateFilterField(Field field, ); } String dbFieldName = filterField.databaseFieldName(); - if (StringUtils.isNullOrEmpty(dbFieldName)) { + if (StringUtils.isEmpty(dbFieldName)) { dbFieldName = field.getName(); } - if (StringUtils.isNullOrEmpty(filterField.databaseTableAlias())) { + if (StringUtils.isEmpty(filterField.databaseTableAlias())) { this.databaseFieldName = dbFieldName; } else { this.databaseFieldName = String.format("%s.%s", diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java index 10ba3cea93..77bf829ba5 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java @@ -17,16 +17,16 @@ package com.epam.pipeline.vo.filter.converters; -import com.amazonaws.util.StringUtils; import com.epam.pipeline.vo.filter.FilterOperandType; import com.epam.pipeline.vo.filter.WrongFilterException; +import org.apache.commons.lang3.StringUtils; import java.util.Map; public abstract class AbstractFilterValueConverter { String clearQuotes(String value) { - if (!StringUtils.isNullOrEmpty(value) && ((value.startsWith("\"") && value.endsWith("\"")) || + if (StringUtils.isNotEmpty(value) && ((value.startsWith("\"") && value.endsWith("\"")) || (value.startsWith("'") && value.endsWith("'")))) { return value.substring(1, value.length() - 1); } diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DateConverter.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DateConverter.java index 4b21b48ab5..ae5f519f50 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DateConverter.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DateConverter.java @@ -17,10 +17,10 @@ package com.epam.pipeline.vo.filter.converters; -import com.amazonaws.util.StringUtils; import com.epam.pipeline.vo.filter.FilterOperandType; import com.epam.pipeline.vo.filter.WrongFilterException; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import java.text.DateFormat; import java.text.ParseException; @@ -57,7 +57,7 @@ public Object convert(String field, if (params != null && params.containsKey(TIMEZONE_OFFSET_PARAMETER)) { timezoneOffset = Integer.parseInt(params.get(TIMEZONE_OFFSET_PARAMETER).toString()); } - if (!StringUtils.isNullOrEmpty(dateValue)) { + if (StringUtils.isNotEmpty(dateValue)) { try { result = dateFormat.parse(dateValue); if (!dateValue.equals(dateFormat.format(result))) { diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index cd6b851414..df560a1467 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -13,6 +13,9 @@ import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.vo.PagingRunFilterExpressionVO; import com.epam.pipeline.vo.RunStatusVO; +import com.epam.pipeline.vo.filter.FilterExpression; +import com.epam.pipeline.vo.filter.FilterExpressionType; +import com.epam.pipeline.vo.filter.FilterOperandType; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; @@ -39,11 +42,9 @@ public class TesTaskServiceImpl implements TesTaskService { private final MessageHelper messageHelper; private static final String ID = "id"; - private static final String NAME_PREFIX = "namePrefix"; - private static final String VIEW = "view"; + private static final String NAME_PREFIX = "pod.id"; private static final String DEFAULT_PAGE_TOKEN = "1"; private static final Long DEFAULT_PAGE_SIZE = 256L; - private static final TaskView DEFAULT_TASK_VIEW = TaskView.BASIC; @Autowired public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient, TaskMapper taskMapper, @@ -67,21 +68,23 @@ public TesCreateTaskResponse submitTesTask(TesTask body) { public TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String pageToken, TaskView view) { TesListTasksResponse tesListTasksResponse = new TesListTasksResponse(); PagingRunFilterExpressionVO filterExpressionVO = new PagingRunFilterExpressionVO(); - filterExpressionVO.setPage(Integer.parseInt(Optional.ofNullable(pageToken).orElse(DEFAULT_PAGE_TOKEN))); filterExpressionVO.setPageSize(Optional.ofNullable(pageSize).orElse(DEFAULT_PAGE_SIZE).intValue()); - filterExpressionVO.getFilterExpression().setField(VIEW); - filterExpressionVO.getFilterExpression().setValue(Optional.ofNullable(view).orElse(DEFAULT_TASK_VIEW).toString()); - Optional.ofNullable(namePrefix).ifPresent((name) -> { - filterExpressionVO.getFilterExpression().setField(NAME_PREFIX); - filterExpressionVO.getFilterExpression().setValue(name); - }); + if (StringUtils.isNotEmpty(namePrefix)) { + FilterExpression expression = new FilterExpression(); + expression.setField(NAME_PREFIX); + expression.setValue(namePrefix); + expression.setOperand(FilterOperandType.EQUALS.getOperand()); + expression.setFilterExpressionType(FilterExpressionType.LOGICAL); + filterExpressionVO.setFilterExpression(expression); + } List pipelineRunList = cloudPipelineAPIClient.searchRuns(filterExpressionVO).getElements(); tesListTasksResponse.setTasks(pipelineRunList.stream().map(taskMapper::mapToTesTask) .collect(Collectors.toList())); return tesListTasksResponse; } + @Override public void stub() { //stubbed method From e5040b4292ec8557da6af2586ae284d39712e736 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Mon, 16 Sep 2019 18:24:02 +0300 Subject: [PATCH 100/194] refactoring TaskView --- .../tesadapter/service/TaskMapper.java | 51 ++++++++----------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index a3bee3c804..0e6fbe9df6 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -219,37 +219,26 @@ public TesTask mapToTesTask(PipelineRun run, TaskView view) { } private TesTask filterTesTaskWithView(PipelineRun run, TaskView view) { - switch (view) { - case MINIMAL: - return TesTask.builder() - .id(String.valueOf(run.getId())) - .state(createTesState(run)) - .build(); - case BASIC: - return TesTask.builder() - .id(String.valueOf(run.getId())) - .name(run.getPodId()) - .resources(createTesResources(run)) - .executors(createListExecutor(run)) - .outputs(createTesOutput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) - .creationTime(run.getStartDate().toString()) - .state(createTesState(run)) - .build(); - case FULL: - return TesTask.builder() - .id(String.valueOf(run.getId())) - .name(run.getPodId()) - .resources(createTesResources(run)) - .executors(createListExecutor(run)) - .inputs(createTesInput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) - .outputs(createTesOutput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) - .creationTime(run.getStartDate().toString()) - .logs(createTesTaskLog(run.getId())) - .state(createTesState(run)) - .build(); - default: - throw new IllegalArgumentException(messageHelper.getMessage(MessageConstants. - ERROR_PARAMETER_REQUIRED)); + final TesTask.TesTaskBuilder tesTask = TesTask.builder() + .id(String.valueOf(run.getId())) + .state(createTesState(run)); + if (view == TaskView.MINIMAL) { + return tesTask.build(); + } + tesTask.name(run.getPodId()) + .resources(createTesResources(run)) + .executors(createListExecutor(run)) + .outputs(createTesOutput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) + .creationTime(run.getStartDate().toString()); + if (view == TaskView.BASIC) { + return tesTask.build(); + } + tesTask.inputs(createTesInput(ListUtils.emptyIfNull(run.getPipelineRunParameters()))) + .logs(createTesTaskLog(run.getId())); + if (view == TaskView.FULL) { + return tesTask.build(); + } else { + throw new IllegalArgumentException(messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_REQUIRED)); } } From c349931430dd1e0431a79bde62dceed0e4ae0fab Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Mon, 16 Sep 2019 18:27:06 +0300 Subject: [PATCH 101/194] refactored listTesTask method, added "filterRuns()" method to make possible filter by default way - if "namePrefix" is empty. --- .../client/pipeline/CloudPipelineAPI.java | 6 ++++ .../com/epam/pipeline/rest/PagedResult.java | 5 +++ .../service/CloudPipelineAPIClient.java | 15 +++++--- .../service/TesTaskServiceImpl.java | 35 +++++++++++++------ 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java index cd537260de..3f88339530 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/client/pipeline/CloudPipelineAPI.java @@ -48,6 +48,7 @@ import com.epam.pipeline.vo.EntityVO; import com.epam.pipeline.vo.FilterNodesVO; import com.epam.pipeline.vo.PagingRunFilterExpressionVO; +import com.epam.pipeline.vo.PagingRunFilterVO; import com.epam.pipeline.vo.RunStatusVO; import com.epam.pipeline.vo.notification.NotificationMessageVO; import okhttp3.MultipartBody; @@ -79,6 +80,7 @@ public interface CloudPipelineAPI { String TOOL_ID = "toolId"; String REGION_ID = "regionId"; String SPOT = "spot"; + String LOAD_LINKS = "loadLinks"; @POST("run") @@ -87,6 +89,10 @@ public interface CloudPipelineAPI { @POST("run/search") Call>>> searchRuns(@Body PagingRunFilterExpressionVO filterVO); + @POST("run/filter") + Call>>> filterRuns(@Body PagingRunFilterVO filterVO, + @Query(LOAD_LINKS) Boolean loadStorageLinks); + @POST("run/{runId}/status") Call> updateRunStatus(@Path(RUN_ID) Long runId, @Body RunStatusVO statusUpdate); diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/rest/PagedResult.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/rest/PagedResult.java index 28681d1f1e..90e05bb7c4 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/rest/PagedResult.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/rest/PagedResult.java @@ -16,6 +16,11 @@ package com.epam.pipeline.rest; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Setter +@NoArgsConstructor public class PagedResult { // TODO: refactor to extend Result class private T elements; // TODO; refactor to contain a list of T private int totalCount; diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index 8f2ff4914f..a244d46edc 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -5,14 +5,15 @@ import com.epam.pipeline.entity.cluster.AllowedInstanceAndPriceTypes; import com.epam.pipeline.entity.datastorage.AbstractDataStorage; import com.epam.pipeline.entity.pipeline.PipelineRun; -import com.epam.pipeline.entity.pipeline.Tool; import com.epam.pipeline.entity.pipeline.PipelineTask; import com.epam.pipeline.entity.pipeline.RunLog; +import com.epam.pipeline.entity.pipeline.Tool; import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.entity.region.AbstractCloudRegion; import com.epam.pipeline.rest.PagedResult; import com.epam.pipeline.utils.QueryUtils; import com.epam.pipeline.vo.PagingRunFilterExpressionVO; +import com.epam.pipeline.vo.PagingRunFilterVO; import com.epam.pipeline.vo.RunStatusVO; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -51,24 +52,28 @@ public Tool loadTool(String image) { return QueryUtils.execute(cloudPipelineAPI.loadTool(null, image)); } - public List loadAllRegions(){ + public List loadAllRegions() { return QueryUtils.execute(cloudPipelineAPI.loadAllRegions()); } - public List loadAllDataStorages(){ + public List loadAllDataStorages() { return QueryUtils.execute(cloudPipelineAPI.loadAllDataStorages()); } - public List getRunLog(final Long runId){ + public List getRunLog(final Long runId) { return QueryUtils.execute(cloudPipelineAPI.loadLogs(runId)); } - public List loadPipelineTasks(final Long id){ + public List loadPipelineTasks(final Long id) { return QueryUtils.execute(cloudPipelineAPI.loadPipelineTasks(id)); } public PagedResult> searchRuns(PagingRunFilterExpressionVO filterVO) { return QueryUtils.execute(cloudPipelineAPI.searchRuns(filterVO)); } + + public PagedResult> filterRuns(PagingRunFilterVO filterVO, Boolean loadStorageLinks) { + return QueryUtils.execute(cloudPipelineAPI.filterRuns(filterVO, loadStorageLinks)); + } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index df560a1467..768fc04d84 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -12,6 +12,7 @@ import com.epam.pipeline.tesadapter.entity.TesServiceInfo; import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.vo.PagingRunFilterExpressionVO; +import com.epam.pipeline.vo.PagingRunFilterVO; import com.epam.pipeline.vo.RunStatusVO; import com.epam.pipeline.vo.filter.FilterExpression; import com.epam.pipeline.vo.filter.FilterExpressionType; @@ -44,6 +45,7 @@ public class TesTaskServiceImpl implements TesTaskService { private static final String ID = "id"; private static final String NAME_PREFIX = "pod.id"; private static final String DEFAULT_PAGE_TOKEN = "1"; + private static final Boolean LOAD_STORAGE_LINKS = true; private static final Long DEFAULT_PAGE_SIZE = 256L; @Autowired @@ -67,23 +69,36 @@ public TesCreateTaskResponse submitTesTask(TesTask body) { @Override public TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String pageToken, TaskView view) { TesListTasksResponse tesListTasksResponse = new TesListTasksResponse(); - PagingRunFilterExpressionVO filterExpressionVO = new PagingRunFilterExpressionVO(); - filterExpressionVO.setPage(Integer.parseInt(Optional.ofNullable(pageToken).orElse(DEFAULT_PAGE_TOKEN))); - filterExpressionVO.setPageSize(Optional.ofNullable(pageSize).orElse(DEFAULT_PAGE_SIZE).intValue()); + List pipelineRunList; if (StringUtils.isNotEmpty(namePrefix)) { - FilterExpression expression = new FilterExpression(); - expression.setField(NAME_PREFIX); - expression.setValue(namePrefix); - expression.setOperand(FilterOperandType.EQUALS.getOperand()); - expression.setFilterExpressionType(FilterExpressionType.LOGICAL); - filterExpressionVO.setFilterExpression(expression); + pipelineRunList = searchRunsWithNamePrefix(namePrefix, pageSize, pageToken); + } else { + pipelineRunList = filterRunsWithOutNamePrefix(pageSize, pageToken); } - List pipelineRunList = cloudPipelineAPIClient.searchRuns(filterExpressionVO).getElements(); tesListTasksResponse.setTasks(pipelineRunList.stream().map(taskMapper::mapToTesTask) .collect(Collectors.toList())); return tesListTasksResponse; } + private List searchRunsWithNamePrefix(String namePrefix, Long pageSize, String pageToken) { + PagingRunFilterExpressionVO filterExpressionVO = new PagingRunFilterExpressionVO(); + FilterExpression expression = new FilterExpression(); + expression.setField(NAME_PREFIX); + expression.setValue(namePrefix); + expression.setOperand(FilterOperandType.EQUALS.getOperand()); + expression.setFilterExpressionType(FilterExpressionType.LOGICAL); + filterExpressionVO.setFilterExpression(expression); + filterExpressionVO.setPage(Integer.parseInt(Optional.ofNullable(pageToken).orElse(DEFAULT_PAGE_TOKEN))); + filterExpressionVO.setPageSize(Optional.ofNullable(pageSize).orElse(DEFAULT_PAGE_SIZE).intValue()); + return cloudPipelineAPIClient.searchRuns(filterExpressionVO).getElements(); + } + + private List filterRunsWithOutNamePrefix(Long pageSize, String pageToken) { + PagingRunFilterVO filterVO = new PagingRunFilterVO(); + filterVO.setPage(Integer.parseInt(Optional.ofNullable(pageToken).orElse(DEFAULT_PAGE_TOKEN))); + filterVO.setPageSize(Optional.ofNullable(pageSize).orElse(DEFAULT_PAGE_SIZE).intValue()); + return cloudPipelineAPIClient.filterRuns(filterVO, LOAD_STORAGE_LINKS).getElements(); + } @Override public void stub() { From d3cb9508661c4a774a043f58d1b6fe5de8c1ec92 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Mon, 16 Sep 2019 18:46:11 +0300 Subject: [PATCH 102/194] added resources.zone and dockerImage --- .../tesadapter/service/CloudPipelineAPIClient.java | 4 ++++ .../com/epam/pipeline/tesadapter/service/TaskMapper.java | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index d4eef5fca8..e399b5a935 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -65,4 +65,8 @@ public List getRunLog(final Long runId){ public List loadPipelineTasks(final Long id){ return QueryUtils.execute(cloudPipelineAPI.loadPipelineTasks(id)); } + + public AbstractCloudRegion loadRegion(final Long regionId){ + return QueryUtils.execute(cloudPipelineAPI.loadRegion(regionId)); + } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index d79a46ac0f..09f6d1829a 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -281,13 +281,19 @@ private TesResources createTesResources(PipelineRun run) { .diskGb(new Double(run.getInstance().getNodeDisk())) .ramGb(getInstanceType(run).getMemory() * convertMemoryUnitTypeToGiB(getInstanceType(run).getMemoryUnit())) .cpuCores((long) getInstanceType(run).getVCPU()) + .zones(getPipelineRunZone(run)) .build(); } + private List getPipelineRunZone(PipelineRun run) { + return Collections.singletonList(cloudPipelineAPIClient.loadRegion(run.getInstance().getCloudRegionId()).getRegionCode()); + } + private List createListExecutor(PipelineRun run) { - return ListUtils.emptyIfNull(Arrays.asList(TesExecutor.builder() + return ListUtils.emptyIfNull(Collections.singletonList(TesExecutor.builder() .command(ListUtils.emptyIfNull(Arrays.asList(run.getActualCmd().split(SEPARATOR)))) .env(run.getEnvVars()) + .image(run.getDockerImage()) .build())); } From 43fa627c80e00ee98c86391a8ac4551c7c2be3ff Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Mon, 16 Sep 2019 19:19:23 +0300 Subject: [PATCH 103/194] refactored two methods: createInput and Output. --- .../pipeline/tesadapter/entity/TesInput.java | 6 +++++ .../pipeline/tesadapter/entity/TesOutput.java | 6 +++++ .../tesadapter/service/TaskMapper.java | 26 +++++++++---------- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java index 6d9d5d5c4c..f88f0e1d96 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesInput.java @@ -3,12 +3,18 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import javax.validation.Valid; @ApiModel(description = "Input describes Task input files.") @Data +@Builder +@NoArgsConstructor +@AllArgsConstructor public class TesInput { @ApiModelProperty(value = "") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java index 38cccd4138..8f04709976 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesOutput.java @@ -3,12 +3,18 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import javax.validation.Valid; @ApiModel(description = "Output describes Task output files.") @Data +@Builder +@NoArgsConstructor +@AllArgsConstructor public class TesOutput { @ApiModelProperty(value = "") @JsonProperty("name") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 09f6d1829a..abb4fae0ea 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -254,25 +254,23 @@ private TesState createTesState(PipelineRun run) { } private List createTesInput(List parameters) { - final TesInput tesInput = new TesInput(); - parameters.stream() + return parameters.stream() .filter(pipelineRunParameter -> pipelineRunParameter.getType().contains(INPUT_TYPE)) - .forEach(pipelineRunParameter -> { - tesInput.setName(pipelineRunParameter.getName()); - tesInput.setUrl(pipelineRunParameter.getValue()); - }); - return ListUtils.emptyIfNull(Arrays.asList(tesInput)); + .map(pipelineRunParameter -> + TesInput.builder() + .name(pipelineRunParameter.getName()) + .url(pipelineRunParameter.getValue()) + .build()).collect(Collectors.toList()); } private List createTesOutput(List parameters) { - final TesOutput tesOutput = new TesOutput(); - parameters.stream() + return parameters.stream() .filter(pipelineRunParameter -> pipelineRunParameter.getType().contains(OUTPUT_TYPE)) - .forEach(pipelineRunParameter -> { - tesOutput.setName(pipelineRunParameter.getName()); - tesOutput.setUrl(pipelineRunParameter.getValue()); - }); - return ListUtils.emptyIfNull(Arrays.asList(tesOutput)); + .map(pipelineRunParameter -> + TesOutput.builder() + .name(pipelineRunParameter.getName()) + .url(pipelineRunParameter.getValue()) + .build()).collect(Collectors.toList()); } private TesResources createTesResources(PipelineRun run) { From 506908e5d02d5d58ed2edfa629ed85b1aa2c9de0 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Mon, 16 Sep 2019 22:54:46 +0300 Subject: [PATCH 104/194] codacy clean-up, added explicit scoping --- .../main/java/com/epam/pipeline/vo/filter/AclExpression.java | 2 +- .../main/java/com/epam/pipeline/vo/filter/FilterExpression.java | 2 +- .../java/com/epam/pipeline/vo/filter/LogicalExpression.java | 2 +- .../vo/filter/converters/AbstractFilterValueConverter.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpression.java index f9e444ebb3..2b26683c37 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpression.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpression.java @@ -47,7 +47,7 @@ class AclExpression extends LogicalExpression { } @Override - boolean filterFieldMatches(FilterField filterField) { + public boolean filterFieldMatches(FilterField filterField) { return filterField.aclField() == this.getAclExpressionType(); } } diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java index 74968d14ea..664324b0cc 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java @@ -69,7 +69,7 @@ public FilterExpression preProcessExpression(Class context, return this; } - static FilterExpression generate(FilterExpression rootExpression, + public static FilterExpression generate(FilterExpression rootExpression, List allowedPipelines, String ownership) throws WrongFilterException { diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java index f692ac314c..188ac2ebb6 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java @@ -230,7 +230,7 @@ private FilterExpression preProcessField(Field field, } } - boolean filterFieldMatches(FilterField filterField) { + public boolean filterFieldMatches(FilterField filterField) { return Pattern.matches(filterField.displayName(), this.getField()) || filterField.displayName().equalsIgnoreCase(this.getField()); } diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java index 77bf829ba5..b562d44d07 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java @@ -25,7 +25,7 @@ public abstract class AbstractFilterValueConverter { - String clearQuotes(String value) { + public String clearQuotes(String value) { if (StringUtils.isNotEmpty(value) && ((value.startsWith("\"") && value.endsWith("\"")) || (value.startsWith("'") && value.endsWith("'")))) { return value.substring(1, value.length() - 1); From aa9293f7cf0255ca59c723de2452b98406887e78 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 17 Sep 2019 14:45:59 +0300 Subject: [PATCH 105/194] refactored listTesTask method, implemented "view" mapping --- .../epam/pipeline/tesadapter/service/TesTaskServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index d82340bc9d..70bcdf4dff 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -75,8 +75,8 @@ public TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String } else { pipelineRunList = filterRunsWithOutNamePrefix(pageSize, pageToken); } - tesListTasksResponse.setTasks(pipelineRunList.stream().map(taskMapper::mapToTesTask) - .collect(Collectors.toList())); + tesListTasksResponse.setTasks(pipelineRunList.stream().map(pipelineRun -> + taskMapper.mapToTesTask(pipelineRun, view)).collect(Collectors.toList())); return tesListTasksResponse; } From cc3888c56cde7d5c2b29c3418745cf2735578d4c Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 17 Sep 2019 15:02:35 +0300 Subject: [PATCH 106/194] refactored listTesTask method, implemented "view" mapping. added default "view" with optional --- .../java/com/epam/pipeline/tesadapter/entity/TaskView.java | 2 -- .../epam/pipeline/tesadapter/service/TesTaskServiceImpl.java | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskView.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskView.java index 432d737dce..8bfe887a6d 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskView.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TaskView.java @@ -1,8 +1,6 @@ package com.epam.pipeline.tesadapter.entity; -import lombok.ToString; -@ToString public enum TaskView { MINIMAL, BASIC, diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 70bcdf4dff..b9664181ee 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -41,7 +41,7 @@ public class TesTaskServiceImpl implements TesTaskService { private final CloudPipelineAPIClient cloudPipelineAPIClient; private final TaskMapper taskMapper; private final MessageHelper messageHelper; - + private static final TaskView DEFAULT_TASK_VIEW = TaskView.MINIMAL; private static final String ID = "id"; private static final String NAME_PREFIX = "pod.id"; private static final String DEFAULT_PAGE_TOKEN = "1"; @@ -76,7 +76,8 @@ public TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String pipelineRunList = filterRunsWithOutNamePrefix(pageSize, pageToken); } tesListTasksResponse.setTasks(pipelineRunList.stream().map(pipelineRun -> - taskMapper.mapToTesTask(pipelineRun, view)).collect(Collectors.toList())); + taskMapper.mapToTesTask(pipelineRun, Optional.ofNullable(view).orElse(DEFAULT_TASK_VIEW))) + .collect(Collectors.toList())); return tesListTasksResponse; } From 7b0d9dfdbba08802cc2dea4f5d0a92e52bb497f5 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 18 Sep 2019 11:42:12 +0300 Subject: [PATCH 107/194] refactored listTesTask method, removed unnecessary classes from "common" module, changed rest classes to VO style. couple modifications in WebMvc-Test class and ServiceImpl class --- .../vo/PagingRunFilterExpressionVO.java | 4 +- .../pipeline/vo/filter/AclExpression.java | 53 ---- .../pipeline/vo/filter/AclExpressionType.java | 51 ---- .../vo/filter/AndFilterExpression.java | 34 --- .../pipeline/vo/filter/FilterExpression.java | 145 --------- ...nType.java => FilterExpressionTypeVO.java} | 15 +- ...Exception.java => FilterExpressionVO.java} | 22 +- .../epam/pipeline/vo/filter/FilterField.java | 51 ---- .../epam/pipeline/vo/filter/FilterFields.java | 29 -- ...randType.java => FilterOperandTypeVO.java} | 9 +- .../pipeline/vo/filter/LogicalExpression.java | 274 ------------------ .../vo/filter/OrFilterExpression.java | 34 --- .../composers/AbstractFilterComposer.java | 25 -- .../vo/filter/composers/DefaultComposer.java | 28 -- .../AbstractFilterValueConverter.java | 41 --- .../vo/filter/converters/DateConverter.java | 97 ------- .../filter/converters/DefaultConverter.java | 35 --- .../service/TesTaskServiceImpl.java | 17 +- .../app/TesAdapterControllerTest.java | 12 + .../service/TesTaskServiceImplTest.java | 3 +- 20 files changed, 55 insertions(+), 924 deletions(-) delete mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpression.java delete mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpressionType.java delete mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AndFilterExpression.java delete mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java rename cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/{FilterExpressionType.java => FilterExpressionTypeVO.java} (78%) rename cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/{WrongFilterException.java => FilterExpressionVO.java} (58%) delete mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterField.java delete mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterFields.java rename cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/{FilterOperandType.java => FilterOperandTypeVO.java} (86%) delete mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java delete mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/OrFilterExpression.java delete mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/AbstractFilterComposer.java delete mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/DefaultComposer.java delete mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java delete mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DateConverter.java delete mode 100644 cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DefaultConverter.java diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java index f8b3e566d1..b17aa08497 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java @@ -17,7 +17,7 @@ package com.epam.pipeline.vo; import com.epam.pipeline.entity.filter.AclSecuredFilter; -import com.epam.pipeline.vo.filter.FilterExpression; +import com.epam.pipeline.vo.filter.FilterExpressionVO; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.NoArgsConstructor; @@ -33,7 +33,7 @@ public class PagingRunFilterExpressionVO implements AclSecuredFilter { private int page; private int pageSize; private int timezoneOffsetInMinutes; - private FilterExpression filterExpression; + private FilterExpressionVO filterExpressionVO; //these filter is used for ACL filtering @JsonIgnore diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpression.java deleted file mode 100644 index 2b26683c37..0000000000 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpression.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.epam.pipeline.vo.filter; - -import lombok.Getter; -import lombok.Setter; - -import java.util.List; -import java.util.stream.Collectors; - -@Getter -@Setter -class AclExpression extends LogicalExpression { - private AclExpressionType aclExpressionType; - - AclExpression(AclExpressionType aclExpressionType, String value) { - super(aclExpressionType.getAclType(), value, FilterOperandType.EQUALS); - this.setAclExpressionType(aclExpressionType); - } - - AclExpression(AclExpressionType aclExpressionType, List value) { - super( - aclExpressionType.getAclType(), - String.join( - ", ", - value - .stream() - .map(Object::toString) - .collect(Collectors.toList()) - ), - FilterOperandType.EQUALS); - this.setAclExpressionType(aclExpressionType); - } - - @Override - public boolean filterFieldMatches(FilterField filterField) { - return filterField.aclField() == this.getAclExpressionType(); - } -} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpressionType.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpressionType.java deleted file mode 100644 index 6f552eec84..0000000000 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AclExpressionType.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.epam.pipeline.vo.filter; - -import java.util.HashMap; -import java.util.Map; - -public enum AclExpressionType { - NONE("none"), - PIPELINE_IDS("pipeline_ids"), - OWNERSHIP("ownership"); - - private String aclType; - - private static Map operandsMap = new HashMap<>(); - static { - operandsMap.put(NONE.aclType, NONE); - operandsMap.put(PIPELINE_IDS.aclType, PIPELINE_IDS); - operandsMap.put(OWNERSHIP.aclType, OWNERSHIP); - } - - AclExpressionType(String type) { - this.aclType = type; - } - - public static AclExpressionType getByString(String type) { - if (type == null) { - return null; - } - return operandsMap.get(type); - } - - - public String getAclType() { - return this.aclType; - } -} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AndFilterExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AndFilterExpression.java deleted file mode 100644 index ea63f0314d..0000000000 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/AndFilterExpression.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.epam.pipeline.vo.filter; - -public class AndFilterExpression extends FilterExpression { - - AndFilterExpression() { - super(); - } - - @Override - public String toSQLStatement() throws WrongFilterException { - if (this.getExpressions() != null && this.getExpressions().size() == 2) { - return String.format("(%s AND %s)", - this.getExpressions().get(0).toSQLStatement(), - this.getExpressions().get(1).toSQLStatement()); - } - throw new WrongFilterException(); - } -} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java deleted file mode 100644 index 664324b0cc..0000000000 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.epam.pipeline.vo.filter; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import org.apache.commons.lang3.StringUtils; -import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -@Getter -@Setter -@NoArgsConstructor -public class FilterExpression { - - private String field; - private String value; - private String operand; - @JsonIgnore - private FilterOperandType operandType; - @JsonIgnore - private FilterField associatedFilterField; - - private List expressions; - private FilterExpressionType filterExpressionType; - - public String toSQLStatement() throws WrongFilterException { - throw new WrongFilterException(); - } - - public FilterExpression preProcessExpression(Class context, - MapSqlParameterSource parameterSource, - Map parametersPlaceholders, - Map params) - throws WrongFilterException { - if (this.expressions != null) { - List childExpressions = new ArrayList<>(); - for (FilterExpression childExpression : this.expressions) { - childExpressions.add( - childExpression.preProcessExpression( - context, - parameterSource, - parametersPlaceholders, - params) - ); - } - this.expressions = childExpressions; - } - return this; - } - - public static FilterExpression generate(FilterExpression rootExpression, - List allowedPipelines, - String ownership) - throws WrongFilterException { - boolean containsOwnershipFilter = StringUtils.isNotEmpty(ownership); - boolean containsPipelineIdsFilter = allowedPipelines != null && allowedPipelines.size() > 0; - if (containsPipelineIdsFilter && containsOwnershipFilter) { - AndFilterExpression mainExpression = new AndFilterExpression(); - OrFilterExpression aclExpression = new OrFilterExpression(); - aclExpression.setExpressions(Arrays.asList( - new AclExpression(AclExpressionType.PIPELINE_IDS, allowedPipelines), - new AclExpression(AclExpressionType.OWNERSHIP, ownership) - )); - mainExpression.setExpressions(Arrays.asList( - FilterExpression.prepare(rootExpression), - aclExpression - )); - return mainExpression; - } else if (containsOwnershipFilter) { - AndFilterExpression mainExpression = new AndFilterExpression(); - mainExpression.setExpressions(Arrays.asList( - FilterExpression.prepare(rootExpression), - new AclExpression(AclExpressionType.OWNERSHIP, ownership) - )); - return mainExpression; - } else if (containsPipelineIdsFilter) { - AndFilterExpression mainExpression = new AndFilterExpression(); - mainExpression.setExpressions(Arrays.asList( - FilterExpression.prepare(rootExpression), - new AclExpression(AclExpressionType.PIPELINE_IDS, allowedPipelines) - )); - return mainExpression; - } else { - return FilterExpression.prepare(rootExpression); - } - } - - private static FilterExpression prepare(FilterExpression expression) throws WrongFilterException { - FilterExpression result = null; - if (expression.getFilterExpressionType() == null) { - throw new WrongFilterException("Unknown expression type"); - } - switch (expression.filterExpressionType) { - case LOGICAL: - if (FilterOperandType.getByString(expression.getOperand()) == null) { - throw new WrongFilterException( - String.format("Unknown or incorrect operand: '%s'", - expression.getOperand()) - ); - } - result = new LogicalExpression( - expression.getField(), - expression.getValue(), - FilterOperandType.getByString(expression.getOperand())); - break; - case OR: - result = new OrFilterExpression(); - break; - case AND: - result = new AndFilterExpression(); - break; - default: - break; - } - if (expression.getExpressions() != null && result != null) { - result.setExpressions(new ArrayList<>()); - for (FilterExpression child : expression.getExpressions()) { - result.getExpressions().add(FilterExpression.prepare(child)); - } - } - return result; - } - -} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionType.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionTypeVO.java similarity index 78% rename from cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionType.java rename to cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionTypeVO.java index d2dc44a67e..cb70fcdaae 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionType.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionTypeVO.java @@ -19,37 +19,40 @@ import java.util.HashMap; import java.util.Map; -public enum FilterExpressionType { +public enum FilterExpressionTypeVO { LOGICAL(0), AND(1), OR(2); private long id; - private static Map idMap = new HashMap<>(); + private static Map idMap = new HashMap<>(); + static { idMap.put(LOGICAL.id, LOGICAL); idMap.put(AND.id, AND); idMap.put(OR.id, OR); } - private static Map namesMap = new HashMap<>(); + + private static Map namesMap = new HashMap<>(); + static { namesMap.put(LOGICAL.name(), LOGICAL); namesMap.put(AND.name(), AND); namesMap.put(OR.name(), OR); } - FilterExpressionType(long id) { + FilterExpressionTypeVO(long id) { this.id = id; } - public static FilterExpressionType getById(Long id) { + public static FilterExpressionTypeVO getById(Long id) { if (id == null) { return null; } return idMap.get(id); } - public static FilterExpressionType getByName(String name) { + public static FilterExpressionTypeVO getByName(String name) { if (name == null) { return null; } diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/WrongFilterException.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionVO.java similarity index 58% rename from cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/WrongFilterException.java rename to cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionVO.java index 34c5eb746d..3f197a7dc2 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/WrongFilterException.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionVO.java @@ -16,13 +16,23 @@ package com.epam.pipeline.vo.filter; -public class WrongFilterException extends Exception { +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; - public WrongFilterException() { +import java.util.List; - } +@Getter +@Setter +@NoArgsConstructor +public class FilterExpressionVO { - public WrongFilterException(String message) { - super(message); - } + private String field; + private String value; + private String operand; + @JsonIgnore + private FilterOperandTypeVO operandTypeVO; + private List expressions; + private FilterExpressionTypeVO filterExpressionTypeVO; } diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterField.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterField.java deleted file mode 100644 index 2d7a6da00c..0000000000 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterField.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.epam.pipeline.vo.filter; - -import com.epam.pipeline.vo.filter.composers.AbstractFilterComposer; -import com.epam.pipeline.vo.filter.composers.DefaultComposer; -import com.epam.pipeline.vo.filter.converters.AbstractFilterValueConverter; -import com.epam.pipeline.vo.filter.converters.DefaultConverter; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Repeatable; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Target({ElementType.METHOD, ElementType.FIELD}) -@Retention(RetentionPolicy.RUNTIME) -@Repeatable(FilterFields.class) -public @interface FilterField { - String displayName() default ""; - String databaseFieldName() default ""; - String databaseTableAlias() default ""; - String description() default ""; - AclExpressionType aclField() default AclExpressionType.NONE; - boolean isRegex() default false; - boolean multiplePlaceholders() default false; - FilterOperandType[] supportedOperands() default { - FilterOperandType.LESS, - FilterOperandType.LESS_OR_EQUALS, - FilterOperandType.EQUALS, - FilterOperandType.NOT_EQUALS, - FilterOperandType.MORE_OR_EQUALS, - FilterOperandType.MORE - }; - Class converter() default DefaultConverter.class; - Class composer() default DefaultComposer.class; -} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterFields.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterFields.java deleted file mode 100644 index 44ce0574a0..0000000000 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterFields.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.epam.pipeline.vo.filter; - - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Target({ElementType.METHOD, ElementType.FIELD}) -@Retention(RetentionPolicy.RUNTIME) -public @interface FilterFields { - FilterField[] value(); -} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandType.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandTypeVO.java similarity index 86% rename from cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandType.java rename to cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandTypeVO.java index b53e3b4d72..7fe01956c2 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandType.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandTypeVO.java @@ -19,7 +19,7 @@ import java.util.HashMap; import java.util.Map; -public enum FilterOperandType { +public enum FilterOperandTypeVO { LESS("<"), LESS_OR_EQUALS("<="), EQUALS("="), @@ -29,7 +29,8 @@ public enum FilterOperandType { private String operand; - private static Map operandsMap = new HashMap<>(); + private static Map operandsMap = new HashMap<>(); + static { operandsMap.put(LESS.operand, LESS); operandsMap.put(LESS_OR_EQUALS.operand, LESS_OR_EQUALS); @@ -39,11 +40,11 @@ public enum FilterOperandType { operandsMap.put(MORE.operand, MORE); } - FilterOperandType(String operand) { + FilterOperandTypeVO(String operand) { this.operand = operand; } - public static FilterOperandType getByString(String operand) { + public static FilterOperandTypeVO getByString(String operand) { if (operand == null) { return null; } diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java deleted file mode 100644 index 188ac2ebb6..0000000000 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/LogicalExpression.java +++ /dev/null @@ -1,274 +0,0 @@ -/* - * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.epam.pipeline.vo.filter; - -import com.epam.pipeline.vo.filter.composers.AbstractFilterComposer; -import com.epam.pipeline.vo.filter.converters.AbstractFilterValueConverter; -import com.epam.pipeline.vo.filter.converters.DateConverter; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; - -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -@Data -@Slf4j -@EqualsAndHashCode(callSuper = false) -public class LogicalExpression extends FilterExpression { - - private String databaseFieldName; - private String databaseValuePlaceholder; - private List databaseValuePlaceholders; - private boolean multiplePlaceholders; - - private AbstractFilterComposer composer; - - private LogicalExpression() { - super(); - } - - LogicalExpression(String field, String value, FilterOperandType operand) { - this(); - this.setField(field); - this.setValue(value); - this.setOperandType(operand); - } - - public String generateSQLStatement() throws WrongFilterException { - if (StringUtils.isEmpty(this.databaseFieldName) - || StringUtils.isEmpty(this.databaseValuePlaceholder)) { - throw new WrongFilterException(); - } - return String.format("(%s %s :%s)", - this.databaseFieldName, - this.getOperandType().getOperand(), - this.databaseValuePlaceholder); - } - - @Override - public String toSQLStatement() throws WrongFilterException { - if (this.composer == null) { - throw new WrongFilterException(); - } - return this.composer.generateExpression(this); - } - - private String getValuePlaceholder(Field field, - Map parametersPlaceholders) { - String placeholder = field.getName(); - if (!parametersPlaceholders.containsKey(placeholder)) { - parametersPlaceholders.put(placeholder, 0L); - } else { - Long placeholderCount = parametersPlaceholders.get(placeholder) + 1; - parametersPlaceholders.replace(placeholder, placeholderCount); - placeholder = String.format("%s_%d", placeholder, placeholderCount); - } - return placeholder; - } - - private void associateFilterField(Field field, - FilterField filterField, - MapSqlParameterSource parameterSource, - Map parametersPlaceholders, - Map params) - throws WrongFilterException { - if (!Arrays.asList(filterField.supportedOperands()).contains(this.getOperandType())) { - final String supportedOperands = String.join( - ", ", - Arrays.stream(filterField.supportedOperands()) - .map(operand -> String.format("'%s'", operand.getOperand())) - .collect(Collectors.toList()) - ); - throw new WrongFilterException( - String.format( - "Operand '%s' is not supported for field '%s'. Supported operands are: %s", - this.getOperandType().getOperand(), - this.getField(), - supportedOperands - ) - ); - } - String dbFieldName = filterField.databaseFieldName(); - if (StringUtils.isEmpty(dbFieldName)) { - dbFieldName = field.getName(); - } - if (StringUtils.isEmpty(filterField.databaseTableAlias())) { - this.databaseFieldName = dbFieldName; - } else { - this.databaseFieldName = String.format("%s.%s", - filterField.databaseTableAlias(), - dbFieldName); - } - - Object value; - AbstractFilterValueConverter converter = null; - try { - Constructor constructor = filterField.converter() - .getConstructor(); - converter = constructor.newInstance(); - } catch (NoSuchMethodException | - IllegalAccessException | - InvocationTargetException | - InstantiationException e) { - log.error(e.getMessage(), e); - } - if (converter != null) { - value = converter.convert( - this.getField(), - this.getValue(), - this.getOperandType(), - params); - } else { - value = this.getValue(); - } - - this.multiplePlaceholders = filterField.multiplePlaceholders() && value instanceof Collection; - if (this.multiplePlaceholders) { - this.databaseValuePlaceholders = new ArrayList<>(); - for (Object valueItem : (Collection) value) { - String placeholder = this.getValuePlaceholder(field, parametersPlaceholders); - this.databaseValuePlaceholders.add(placeholder); - parameterSource.addValue(placeholder, valueItem); - } - } else { - this.databaseValuePlaceholder = this.getValuePlaceholder(field, parametersPlaceholders); - parameterSource.addValue(this.databaseValuePlaceholder, value); - } - - try { - Constructor constructor = filterField.composer() - .getConstructor(); - this.composer = constructor.newInstance(); - } catch (NoSuchMethodException | - IllegalAccessException | - InvocationTargetException | - InstantiationException e) { - log.error(e.getMessage(), e); - } - this.setAssociatedFilterField(filterField); - } - - private FilterExpression preProcessField(Field field, - FilterField filterField, - Class context, - MapSqlParameterSource parameterSource, - Map parametersPlaceholders, - Map params) - throws WrongFilterException { - if (DateConverter.class.isAssignableFrom(filterField.converter()) && - ( - this.getOperandType() == FilterOperandType.EQUALS || - this.getOperandType() == FilterOperandType.NOT_EQUALS - )) { - if (this.getOperandType() == FilterOperandType.EQUALS) { - AndFilterExpression andFilterExpression = new AndFilterExpression(); - List expressionList = new ArrayList<>(); - LogicalExpression leftExpression = new LogicalExpression( - this.getField(), - this.getValue(), - FilterOperandType.MORE_OR_EQUALS); - LogicalExpression rightExpression = new LogicalExpression( - this.getField(), - this.getValue(), - FilterOperandType.LESS_OR_EQUALS); - expressionList.add(leftExpression); - expressionList.add(rightExpression); - andFilterExpression.setExpressions(expressionList); - return andFilterExpression.preProcessExpression( - context, - parameterSource, - parametersPlaceholders, - params); - } else { - OrFilterExpression orFilterExpression = new OrFilterExpression(); - List expressionList = new ArrayList<>(); - LogicalExpression leftExpression = new LogicalExpression( - this.getField(), - this.getValue(), - FilterOperandType.MORE); - LogicalExpression rightExpression = new LogicalExpression( - this.getField(), - this.getValue(), - FilterOperandType.LESS); - expressionList.add(leftExpression); - expressionList.add(rightExpression); - orFilterExpression.setExpressions(expressionList); - return orFilterExpression.preProcessExpression( - context, - parameterSource, - parametersPlaceholders, - params); - } - } else { - this.associateFilterField(field, filterField, parameterSource, parametersPlaceholders, params); - return this; - } - } - - public boolean filterFieldMatches(FilterField filterField) { - return Pattern.matches(filterField.displayName(), this.getField()) || - filterField.displayName().equalsIgnoreCase(this.getField()); - } - - @Override - public FilterExpression preProcessExpression(Class context, - MapSqlParameterSource parameterSource, - Map parametersPlaceholders, - Map params) - throws WrongFilterException { - Field[] fields = context.getFields(); - for (Field field : fields) { - FilterFields filterFields = field.getAnnotation(FilterFields.class); - if (filterFields != null) { - for (FilterField filterField : filterFields.value()) { - if (this.filterFieldMatches(filterField)) { - return this.preProcessField( - field, - filterField, - context, - parameterSource, - parametersPlaceholders, - params); - } - } - } else { - FilterField filterField = field.getAnnotation(FilterField.class); - if (filterField != null && this.filterFieldMatches(filterField)) { - return this.preProcessField( - field, - filterField, - context, - parameterSource, - parametersPlaceholders, - params); - } - } - } - throw new WrongFilterException(String.format("Unknown field: %s", this.getField())); - } -} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/OrFilterExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/OrFilterExpression.java deleted file mode 100644 index 7db4fc597f..0000000000 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/OrFilterExpression.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.epam.pipeline.vo.filter; - -public class OrFilterExpression extends FilterExpression { - - OrFilterExpression() { - super(); - } - - @Override - public String toSQLStatement() throws WrongFilterException { - if (this.getExpressions() != null && this.getExpressions().size() == 2) { - return String.format("(%s OR %s)", - this.getExpressions().get(0).toSQLStatement(), - this.getExpressions().get(1).toSQLStatement()); - } - throw new WrongFilterException(); - } -} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/AbstractFilterComposer.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/AbstractFilterComposer.java deleted file mode 100644 index 54579eaec4..0000000000 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/AbstractFilterComposer.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.epam.pipeline.vo.filter.composers; - - -import com.epam.pipeline.vo.filter.LogicalExpression; -import com.epam.pipeline.vo.filter.WrongFilterException; - -public abstract class AbstractFilterComposer { - public abstract String generateExpression(LogicalExpression expression) throws WrongFilterException; -} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/DefaultComposer.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/DefaultComposer.java deleted file mode 100644 index 7492f17e61..0000000000 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/composers/DefaultComposer.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.epam.pipeline.vo.filter.composers; - - -import com.epam.pipeline.vo.filter.LogicalExpression; -import com.epam.pipeline.vo.filter.WrongFilterException; - -public class DefaultComposer extends AbstractFilterComposer { - @Override - public String generateExpression(LogicalExpression expression) throws WrongFilterException { - return expression.generateSQLStatement(); - } -} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java deleted file mode 100644 index b562d44d07..0000000000 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/AbstractFilterValueConverter.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.epam.pipeline.vo.filter.converters; - - -import com.epam.pipeline.vo.filter.FilterOperandType; -import com.epam.pipeline.vo.filter.WrongFilterException; -import org.apache.commons.lang3.StringUtils; - -import java.util.Map; - -public abstract class AbstractFilterValueConverter { - - public String clearQuotes(String value) { - if (StringUtils.isNotEmpty(value) && ((value.startsWith("\"") && value.endsWith("\"")) || - (value.startsWith("'") && value.endsWith("'")))) { - return value.substring(1, value.length() - 1); - } - return value; - } - - public abstract Object convert(String field, - String value, - FilterOperandType operandType, - Map params) - throws WrongFilterException; -} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DateConverter.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DateConverter.java deleted file mode 100644 index ae5f519f50..0000000000 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DateConverter.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.epam.pipeline.vo.filter.converters; - - -import com.epam.pipeline.vo.filter.FilterOperandType; -import com.epam.pipeline.vo.filter.WrongFilterException; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.Map; -import java.util.TimeZone; - -@Slf4j -public class DateConverter extends AbstractFilterValueConverter { - - public static final String TIMEZONE_OFFSET_PARAMETER = "Timezone offset"; - - private static final String DATE_FORMAT = "yyyy-MM-dd"; - - private DateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT); - private Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - - private static final int END_OF_DAY_HOURS = 23; - private static final int END_OF_DAY_MINUTES = 59; - private static final int END_OF_DAY_SECONDS = 59; - private static final int END_OF_DAY_MILLISECONDS = 999; - - @Override - public Object convert(String field, - String value, - FilterOperandType operandType, - Map params) - throws WrongFilterException { - Date result = null; - String dateValue = this.clearQuotes(value); - int timezoneOffset = 0; - if (params != null && params.containsKey(TIMEZONE_OFFSET_PARAMETER)) { - timezoneOffset = Integer.parseInt(params.get(TIMEZONE_OFFSET_PARAMETER).toString()); - } - if (StringUtils.isNotEmpty(dateValue)) { - try { - result = dateFormat.parse(dateValue); - if (!dateValue.equals(dateFormat.format(result))) { - result = null; - } else { - this.calendar.setTime(result); - switch (operandType) { - case LESS_OR_EQUALS: - case MORE: - this.calendar.set(Calendar.HOUR, END_OF_DAY_HOURS); - this.calendar.set(Calendar.MINUTE, END_OF_DAY_MINUTES); - this.calendar.set(Calendar.SECOND, END_OF_DAY_SECONDS); - this.calendar.set(Calendar.MILLISECOND, END_OF_DAY_MILLISECONDS); - break; - default: - this.calendar.set(Calendar.HOUR, 0); - this.calendar.set(Calendar.MINUTE, 0); - this.calendar.set(Calendar.SECOND, 0); - this.calendar.set(Calendar.MILLISECOND, 0); - break; - } - this.calendar.add(Calendar.MINUTE, -timezoneOffset); - result = this.calendar.getTime(); - } - } catch (ParseException e) { - log.error(e.getMessage(), e); - } - if (result != null) { - return result; - } - } - throw new WrongFilterException(String.format( - "Wrong date format (%s). Allowed format is: %s", - value, - DATE_FORMAT)); - } -} diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DefaultConverter.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DefaultConverter.java deleted file mode 100644 index 3fc9e2771f..0000000000 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/converters/DefaultConverter.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.epam.pipeline.vo.filter.converters; - - -import com.epam.pipeline.vo.filter.FilterOperandType; -import com.epam.pipeline.vo.filter.WrongFilterException; - -import java.util.Map; - -public class DefaultConverter extends AbstractFilterValueConverter { - - @Override - public Object convert(String field, - String value, - FilterOperandType operandType, - Map params) - throws WrongFilterException { - return this.clearQuotes(value); - } -} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index b9664181ee..1543efd2a9 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -1,6 +1,7 @@ package com.epam.pipeline.tesadapter.service; +import com.epam.pipeline.entity.datastorage.AbstractDataStorage; import com.epam.pipeline.entity.pipeline.PipelineRun; import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.tesadapter.common.MessageConstants; @@ -14,9 +15,9 @@ import com.epam.pipeline.vo.PagingRunFilterExpressionVO; import com.epam.pipeline.vo.PagingRunFilterVO; import com.epam.pipeline.vo.RunStatusVO; -import com.epam.pipeline.vo.filter.FilterExpression; -import com.epam.pipeline.vo.filter.FilterExpressionType; -import com.epam.pipeline.vo.filter.FilterOperandType; +import com.epam.pipeline.vo.filter.FilterExpressionTypeVO; +import com.epam.pipeline.vo.filter.FilterExpressionVO; +import com.epam.pipeline.vo.filter.FilterOperandTypeVO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; @@ -83,12 +84,12 @@ public TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String private List searchRunsWithNamePrefix(String namePrefix, Long pageSize, String pageToken) { PagingRunFilterExpressionVO filterExpressionVO = new PagingRunFilterExpressionVO(); - FilterExpression expression = new FilterExpression(); + FilterExpressionVO expression = new FilterExpressionVO(); expression.setField(NAME_PREFIX); expression.setValue(namePrefix); - expression.setOperand(FilterOperandType.EQUALS.getOperand()); - expression.setFilterExpressionType(FilterExpressionType.LOGICAL); - filterExpressionVO.setFilterExpression(expression); + expression.setOperand(FilterOperandTypeVO.EQUALS.getOperand()); + expression.setFilterExpressionTypeVO(FilterExpressionTypeVO.LOGICAL); + filterExpressionVO.setFilterExpressionVO(expression); filterExpressionVO.setPage(Integer.parseInt(Optional.ofNullable(pageToken).orElse(DEFAULT_PAGE_TOKEN))); filterExpressionVO.setPageSize(Optional.ofNullable(pageSize).orElse(DEFAULT_PAGE_SIZE).intValue()); return cloudPipelineAPIClient.searchRuns(filterExpressionVO).getElements(); @@ -136,7 +137,7 @@ public TesServiceInfo getServiceInfo() { private List getDataStorage() { return ListUtils.emptyIfNull(cloudPipelineAPIClient.loadAllDataStorages()) - .stream().map(storage -> storage.getPath()) + .stream().map(AbstractDataStorage::getPath) .collect(Collectors.toList()); } } diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index ba39db53ff..95a14852a3 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -6,6 +6,7 @@ import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; +import com.epam.pipeline.tesadapter.entity.TesServiceInfo; import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.tesadapter.service.CloudPipelineAPIClient; import com.epam.pipeline.tesadapter.service.TesTaskServiceImpl; @@ -58,6 +59,17 @@ void submitTesTaskWhenRequestingTesTaskBodyAndReturnId() throws Exception { .andDo(print()).andExpect(status().isOk()).andExpect(content().json(STUBBED_SUBMIT_JSON_RESPONSE)); } + @Test + void expectIllegalArgExceptionWhenRunSubmitTesTasWithNullId() throws Exception { + TesTask tesTask = new TesTask(); + tesTask.setExecutors(null); + when(tesTaskService.submitTesTask(tesTask)).thenThrow(new IllegalArgumentException()); + this.mockMvc.perform(post("/v1/tasks") + .contentType("application/json") + .content(STUBBED_SUBMIT_JSON_REQUEST)) + .andDo(print()).andExpect(status().isInternalServerError()); + } + @Test void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { when(tesTaskService.cancelTesTask(STUBBED_TASK_ID)).thenReturn(new TesCancelTaskResponse()); diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java index 5c5ce9b74f..395a487823 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java @@ -4,6 +4,7 @@ import com.epam.pipeline.rest.PagedResult; import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.entity.TaskView; +import com.epam.pipeline.vo.PagingRunFilterExpressionVO; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -29,7 +30,7 @@ class TesTaskServiceImplTest { @BeforeEach public void setUp() { - when(cloudPipelineAPIClient.searchRuns(any())).thenReturn(pagedPipelineRunList); + when(cloudPipelineAPIClient.searchRuns(any(PagingRunFilterExpressionVO.class))).thenReturn(pagedPipelineRunList); tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient, mock(TaskMapper.class), mock(MessageHelper.class)); } From d645ea6bb6fff894f3c2713f4b8ac11a1b32a285 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 18 Sep 2019 12:06:32 +0300 Subject: [PATCH 108/194] fixed problem related with wrong class names in "common"-module, removed postfix-VO from these classes. --- .../pipeline/vo/PagingRunFilterExpressionVO.java | 4 ++-- ...lterExpressionVO.java => FilterExpression.java} | 10 ++++++---- ...essionTypeVO.java => FilterExpressionType.java} | 12 ++++++------ ...erOperandTypeVO.java => FilterOperandType.java} | 8 ++++---- .../tesadapter/service/TesTaskServiceImpl.java | 14 +++++++------- 5 files changed, 25 insertions(+), 23 deletions(-) rename cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/{FilterExpressionVO.java => FilterExpression.java} (83%) rename cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/{FilterExpressionTypeVO.java => FilterExpressionType.java} (78%) rename cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/{FilterOperandTypeVO.java => FilterOperandType.java} (86%) diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java index b17aa08497..f8b3e566d1 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java @@ -17,7 +17,7 @@ package com.epam.pipeline.vo; import com.epam.pipeline.entity.filter.AclSecuredFilter; -import com.epam.pipeline.vo.filter.FilterExpressionVO; +import com.epam.pipeline.vo.filter.FilterExpression; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.NoArgsConstructor; @@ -33,7 +33,7 @@ public class PagingRunFilterExpressionVO implements AclSecuredFilter { private int page; private int pageSize; private int timezoneOffsetInMinutes; - private FilterExpressionVO filterExpressionVO; + private FilterExpression filterExpression; //these filter is used for ACL filtering @JsonIgnore diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionVO.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java similarity index 83% rename from cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionVO.java rename to cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java index 3f197a7dc2..4f400acfe7 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionVO.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java @@ -26,13 +26,15 @@ @Getter @Setter @NoArgsConstructor -public class FilterExpressionVO { +public class FilterExpression { private String field; private String value; private String operand; + @JsonIgnore - private FilterOperandTypeVO operandTypeVO; - private List expressions; - private FilterExpressionTypeVO filterExpressionTypeVO; + private FilterOperandType operandTypeVO; + + private List expressions; + private FilterExpressionType filterExpressionType; } diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionTypeVO.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionType.java similarity index 78% rename from cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionTypeVO.java rename to cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionType.java index cb70fcdaae..df74e0f67b 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionTypeVO.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionType.java @@ -19,13 +19,13 @@ import java.util.HashMap; import java.util.Map; -public enum FilterExpressionTypeVO { +public enum FilterExpressionType { LOGICAL(0), AND(1), OR(2); private long id; - private static Map idMap = new HashMap<>(); + private static Map idMap = new HashMap<>(); static { idMap.put(LOGICAL.id, LOGICAL); @@ -33,7 +33,7 @@ public enum FilterExpressionTypeVO { idMap.put(OR.id, OR); } - private static Map namesMap = new HashMap<>(); + private static Map namesMap = new HashMap<>(); static { namesMap.put(LOGICAL.name(), LOGICAL); @@ -41,18 +41,18 @@ public enum FilterExpressionTypeVO { namesMap.put(OR.name(), OR); } - FilterExpressionTypeVO(long id) { + FilterExpressionType(long id) { this.id = id; } - public static FilterExpressionTypeVO getById(Long id) { + public static FilterExpressionType getById(Long id) { if (id == null) { return null; } return idMap.get(id); } - public static FilterExpressionTypeVO getByName(String name) { + public static FilterExpressionType getByName(String name) { if (name == null) { return null; } diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandTypeVO.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandType.java similarity index 86% rename from cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandTypeVO.java rename to cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandType.java index 7fe01956c2..322ae1a939 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandTypeVO.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandType.java @@ -19,7 +19,7 @@ import java.util.HashMap; import java.util.Map; -public enum FilterOperandTypeVO { +public enum FilterOperandType { LESS("<"), LESS_OR_EQUALS("<="), EQUALS("="), @@ -29,7 +29,7 @@ public enum FilterOperandTypeVO { private String operand; - private static Map operandsMap = new HashMap<>(); + private static Map operandsMap = new HashMap<>(); static { operandsMap.put(LESS.operand, LESS); @@ -40,11 +40,11 @@ public enum FilterOperandTypeVO { operandsMap.put(MORE.operand, MORE); } - FilterOperandTypeVO(String operand) { + FilterOperandType(String operand) { this.operand = operand; } - public static FilterOperandTypeVO getByString(String operand) { + public static FilterOperandType getByString(String operand) { if (operand == null) { return null; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 1543efd2a9..0f12047726 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -15,9 +15,9 @@ import com.epam.pipeline.vo.PagingRunFilterExpressionVO; import com.epam.pipeline.vo.PagingRunFilterVO; import com.epam.pipeline.vo.RunStatusVO; -import com.epam.pipeline.vo.filter.FilterExpressionTypeVO; -import com.epam.pipeline.vo.filter.FilterExpressionVO; -import com.epam.pipeline.vo.filter.FilterOperandTypeVO; +import com.epam.pipeline.vo.filter.FilterExpressionType; +import com.epam.pipeline.vo.filter.FilterExpression; +import com.epam.pipeline.vo.filter.FilterOperandType; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; @@ -84,12 +84,12 @@ public TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String private List searchRunsWithNamePrefix(String namePrefix, Long pageSize, String pageToken) { PagingRunFilterExpressionVO filterExpressionVO = new PagingRunFilterExpressionVO(); - FilterExpressionVO expression = new FilterExpressionVO(); + FilterExpression expression = new FilterExpression(); expression.setField(NAME_PREFIX); expression.setValue(namePrefix); - expression.setOperand(FilterOperandTypeVO.EQUALS.getOperand()); - expression.setFilterExpressionTypeVO(FilterExpressionTypeVO.LOGICAL); - filterExpressionVO.setFilterExpressionVO(expression); + expression.setOperand(FilterOperandType.EQUALS.getOperand()); + expression.setFilterExpressionType(FilterExpressionType.LOGICAL); + filterExpressionVO.setFilterExpression(expression); filterExpressionVO.setPage(Integer.parseInt(Optional.ofNullable(pageToken).orElse(DEFAULT_PAGE_TOKEN))); filterExpressionVO.setPageSize(Optional.ofNullable(pageSize).orElse(DEFAULT_PAGE_SIZE).intValue()); return cloudPipelineAPIClient.searchRuns(filterExpressionVO).getElements(); From 24cc184cb4c925a97fdbe84e4bd1d7ae9a8d49e1 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 18 Sep 2019 12:12:47 +0300 Subject: [PATCH 109/194] returned post-fix VO to corresponding classes in "common"-module --- .../pipeline/vo/PagingRunFilterExpressionVO.java | 4 ++-- ...pressionType.java => FilterExpressionTypeVO.java} | 12 ++++++------ ...FilterExpression.java => FilterExpressionVO.java} | 8 ++++---- ...lterOperandType.java => FilterOperandTypeVO.java} | 8 ++++---- .../tesadapter/service/TesTaskServiceImpl.java | 12 ++++++------ 5 files changed, 22 insertions(+), 22 deletions(-) rename cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/{FilterExpressionType.java => FilterExpressionTypeVO.java} (78%) rename cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/{FilterExpression.java => FilterExpressionVO.java} (83%) rename cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/{FilterOperandType.java => FilterOperandTypeVO.java} (86%) diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java index f8b3e566d1..9076d0f8fb 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/PagingRunFilterExpressionVO.java @@ -17,7 +17,7 @@ package com.epam.pipeline.vo; import com.epam.pipeline.entity.filter.AclSecuredFilter; -import com.epam.pipeline.vo.filter.FilterExpression; +import com.epam.pipeline.vo.filter.FilterExpressionVO; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.NoArgsConstructor; @@ -33,7 +33,7 @@ public class PagingRunFilterExpressionVO implements AclSecuredFilter { private int page; private int pageSize; private int timezoneOffsetInMinutes; - private FilterExpression filterExpression; + private FilterExpressionVO filterExpression; //these filter is used for ACL filtering @JsonIgnore diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionType.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionTypeVO.java similarity index 78% rename from cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionType.java rename to cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionTypeVO.java index df74e0f67b..cb70fcdaae 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionType.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionTypeVO.java @@ -19,13 +19,13 @@ import java.util.HashMap; import java.util.Map; -public enum FilterExpressionType { +public enum FilterExpressionTypeVO { LOGICAL(0), AND(1), OR(2); private long id; - private static Map idMap = new HashMap<>(); + private static Map idMap = new HashMap<>(); static { idMap.put(LOGICAL.id, LOGICAL); @@ -33,7 +33,7 @@ public enum FilterExpressionType { idMap.put(OR.id, OR); } - private static Map namesMap = new HashMap<>(); + private static Map namesMap = new HashMap<>(); static { namesMap.put(LOGICAL.name(), LOGICAL); @@ -41,18 +41,18 @@ public enum FilterExpressionType { namesMap.put(OR.name(), OR); } - FilterExpressionType(long id) { + FilterExpressionTypeVO(long id) { this.id = id; } - public static FilterExpressionType getById(Long id) { + public static FilterExpressionTypeVO getById(Long id) { if (id == null) { return null; } return idMap.get(id); } - public static FilterExpressionType getByName(String name) { + public static FilterExpressionTypeVO getByName(String name) { if (name == null) { return null; } diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionVO.java similarity index 83% rename from cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java rename to cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionVO.java index 4f400acfe7..f67e52ad89 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpression.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionVO.java @@ -26,15 +26,15 @@ @Getter @Setter @NoArgsConstructor -public class FilterExpression { +public class FilterExpressionVO { private String field; private String value; private String operand; @JsonIgnore - private FilterOperandType operandTypeVO; + private FilterOperandTypeVO operandType; - private List expressions; - private FilterExpressionType filterExpressionType; + private List expressions; + private FilterExpressionTypeVO filterExpressionType; } diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandType.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandTypeVO.java similarity index 86% rename from cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandType.java rename to cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandTypeVO.java index 322ae1a939..7fe01956c2 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandType.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandTypeVO.java @@ -19,7 +19,7 @@ import java.util.HashMap; import java.util.Map; -public enum FilterOperandType { +public enum FilterOperandTypeVO { LESS("<"), LESS_OR_EQUALS("<="), EQUALS("="), @@ -29,7 +29,7 @@ public enum FilterOperandType { private String operand; - private static Map operandsMap = new HashMap<>(); + private static Map operandsMap = new HashMap<>(); static { operandsMap.put(LESS.operand, LESS); @@ -40,11 +40,11 @@ public enum FilterOperandType { operandsMap.put(MORE.operand, MORE); } - FilterOperandType(String operand) { + FilterOperandTypeVO(String operand) { this.operand = operand; } - public static FilterOperandType getByString(String operand) { + public static FilterOperandTypeVO getByString(String operand) { if (operand == null) { return null; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 0f12047726..e795382703 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -15,9 +15,9 @@ import com.epam.pipeline.vo.PagingRunFilterExpressionVO; import com.epam.pipeline.vo.PagingRunFilterVO; import com.epam.pipeline.vo.RunStatusVO; -import com.epam.pipeline.vo.filter.FilterExpressionType; -import com.epam.pipeline.vo.filter.FilterExpression; -import com.epam.pipeline.vo.filter.FilterOperandType; +import com.epam.pipeline.vo.filter.FilterExpressionTypeVO; +import com.epam.pipeline.vo.filter.FilterExpressionVO; +import com.epam.pipeline.vo.filter.FilterOperandTypeVO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; @@ -84,11 +84,11 @@ public TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String private List searchRunsWithNamePrefix(String namePrefix, Long pageSize, String pageToken) { PagingRunFilterExpressionVO filterExpressionVO = new PagingRunFilterExpressionVO(); - FilterExpression expression = new FilterExpression(); + FilterExpressionVO expression = new FilterExpressionVO(); expression.setField(NAME_PREFIX); expression.setValue(namePrefix); - expression.setOperand(FilterOperandType.EQUALS.getOperand()); - expression.setFilterExpressionType(FilterExpressionType.LOGICAL); + expression.setOperand(FilterOperandTypeVO.EQUALS.getOperand()); + expression.setFilterExpressionType(FilterExpressionTypeVO.LOGICAL); filterExpressionVO.setFilterExpression(expression); filterExpressionVO.setPage(Integer.parseInt(Optional.ofNullable(pageToken).orElse(DEFAULT_PAGE_TOKEN))); filterExpressionVO.setPageSize(Optional.ofNullable(pageSize).orElse(DEFAULT_PAGE_SIZE).intValue()); From f99b564e33b5e3be82749513f6a07d285439fe5b Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 18 Sep 2019 12:56:44 +0300 Subject: [PATCH 110/194] couple clean-code in "common"-module and ServiceImpl class --- .../vo/filter/FilterExpressionTypeVO.java | 32 ------------------- .../vo/filter/FilterOperandTypeVO.java | 22 ------------- .../entity/TesListTasksResponse.java | 2 ++ .../service/TesTaskServiceImpl.java | 13 +++----- .../app/TesAdapterControllerTest.java | 1 + 5 files changed, 7 insertions(+), 63 deletions(-) diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionTypeVO.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionTypeVO.java index cb70fcdaae..330fedfdba 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionTypeVO.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterExpressionTypeVO.java @@ -16,49 +16,17 @@ package com.epam.pipeline.vo.filter; -import java.util.HashMap; -import java.util.Map; - public enum FilterExpressionTypeVO { LOGICAL(0), AND(1), OR(2); private long id; - private static Map idMap = new HashMap<>(); - - static { - idMap.put(LOGICAL.id, LOGICAL); - idMap.put(AND.id, AND); - idMap.put(OR.id, OR); - } - - private static Map namesMap = new HashMap<>(); - - static { - namesMap.put(LOGICAL.name(), LOGICAL); - namesMap.put(AND.name(), AND); - namesMap.put(OR.name(), OR); - } FilterExpressionTypeVO(long id) { this.id = id; } - public static FilterExpressionTypeVO getById(Long id) { - if (id == null) { - return null; - } - return idMap.get(id); - } - - public static FilterExpressionTypeVO getByName(String name) { - if (name == null) { - return null; - } - return namesMap.get(name); - } - public Long getId() { return this.id; } diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandTypeVO.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandTypeVO.java index 7fe01956c2..49fc911e24 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandTypeVO.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/vo/filter/FilterOperandTypeVO.java @@ -16,9 +16,6 @@ package com.epam.pipeline.vo.filter; -import java.util.HashMap; -import java.util.Map; - public enum FilterOperandTypeVO { LESS("<"), LESS_OR_EQUALS("<="), @@ -29,29 +26,10 @@ public enum FilterOperandTypeVO { private String operand; - private static Map operandsMap = new HashMap<>(); - - static { - operandsMap.put(LESS.operand, LESS); - operandsMap.put(LESS_OR_EQUALS.operand, LESS_OR_EQUALS); - operandsMap.put(EQUALS.operand, EQUALS); - operandsMap.put(NOT_EQUALS.operand, NOT_EQUALS); - operandsMap.put(MORE_OR_EQUALS.operand, MORE_OR_EQUALS); - operandsMap.put(MORE.operand, MORE); - } - FilterOperandTypeVO(String operand) { this.operand = operand; } - public static FilterOperandTypeVO getByString(String operand) { - if (operand == null) { - return null; - } - return operandsMap.get(operand); - } - - public String getOperand() { return this.operand; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java index cba3480f38..a82c5d8899 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java @@ -4,6 +4,7 @@ import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.NonNull; import javax.validation.Valid; import java.util.List; @@ -14,6 +15,7 @@ public class TesListTasksResponse { @ApiModelProperty(value = "List of tasks.") @JsonProperty("tasks") @Valid + @NonNull private List tasks; @ApiModelProperty(value = "Token used to return the next page of results. See TaskListRequest.next_page_token") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index e795382703..db940ed031 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -69,17 +69,12 @@ public TesCreateTaskResponse submitTesTask(TesTask body) { @Override public TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String pageToken, TaskView view) { - TesListTasksResponse tesListTasksResponse = new TesListTasksResponse(); - List pipelineRunList; - if (StringUtils.isNotEmpty(namePrefix)) { - pipelineRunList = searchRunsWithNamePrefix(namePrefix, pageSize, pageToken); - } else { - pipelineRunList = filterRunsWithOutNamePrefix(pageSize, pageToken); - } - tesListTasksResponse.setTasks(pipelineRunList.stream().map(pipelineRun -> + List pipelineRunList = StringUtils.isNotEmpty(namePrefix) ? + searchRunsWithNamePrefix(namePrefix, pageSize, pageToken) + : filterRunsWithOutNamePrefix(pageSize, pageToken); + return new TesListTasksResponse(pipelineRunList.stream().map(pipelineRun -> taskMapper.mapToTesTask(pipelineRun, Optional.ofNullable(view).orElse(DEFAULT_TASK_VIEW))) .collect(Collectors.toList())); - return tesListTasksResponse; } private List searchRunsWithNamePrefix(String namePrefix, Long pageSize, String pageToken) { diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 95a14852a3..b434b51b3a 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -20,6 +20,7 @@ import org.springframework.test.web.servlet.MockMvc; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; From 2b5f8614d76a0a8c53958cfeea459586d46ebd93 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 18 Sep 2019 13:04:16 +0300 Subject: [PATCH 111/194] couple refactoring in ServiceImpl class for listTesTask method --- .../epam/pipeline/tesadapter/service/TesTaskServiceImpl.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index db940ed031..953b5391f7 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -69,10 +69,9 @@ public TesCreateTaskResponse submitTesTask(TesTask body) { @Override public TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String pageToken, TaskView view) { - List pipelineRunList = StringUtils.isNotEmpty(namePrefix) ? + return new TesListTasksResponse((StringUtils.isNotEmpty(namePrefix) ? searchRunsWithNamePrefix(namePrefix, pageSize, pageToken) - : filterRunsWithOutNamePrefix(pageSize, pageToken); - return new TesListTasksResponse(pipelineRunList.stream().map(pipelineRun -> + : filterRunsWithOutNamePrefix(pageSize, pageToken)).stream().map(pipelineRun -> taskMapper.mapToTesTask(pipelineRun, Optional.ofNullable(view).orElse(DEFAULT_TASK_VIEW))) .collect(Collectors.toList())); } From 9509716cc85dbe46cdb8ca8443e4a0a5b082e006 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 18 Sep 2019 13:29:57 +0300 Subject: [PATCH 112/194] fixed Web Mvc Test, added mock dor TesListTasksResponse --- .../app/TesAdapterControllerTest.java | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index b434b51b3a..eb91344414 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -10,13 +10,11 @@ import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.tesadapter.service.CloudPipelineAPIClient; import com.epam.pipeline.tesadapter.service.TesTaskServiceImpl; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.http.MediaType; -import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.servlet.MockMvc; import static org.mockito.ArgumentMatchers.any; @@ -29,7 +27,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -@ExtendWith(SpringExtension.class) @WebMvcTest(TesAdapterController.class) @SuppressWarnings({"unused", "PMD.TooManyStaticImports"}) public class TesAdapterControllerTest { @@ -37,7 +34,7 @@ public class TesAdapterControllerTest { private static final String PAGE_TOKEN = "1"; private static final String NAME_PREFIX = "pipeline"; private static final Long PAGE_SIZE = 20L; - private static final TaskView DEFAULT_VIEW = TaskView.BASIC; + private static final TaskView DEFAULT_VIEW = TaskView.MINIMAL; private static final String STUBBED_SUBMIT_JSON_REQUEST = "{}"; private static final String STUBBED_SUBMIT_JSON_RESPONSE = "{\"id\":\"5\"}"; @@ -50,12 +47,23 @@ public class TesAdapterControllerTest { @MockBean private CloudPipelineAPIClient cloudPipelineAPIClient; + private TesCreateTaskResponse tesCreateTaskResponse = new TesCreateTaskResponse(); + private TesServiceInfo tesServiceInfo = new TesServiceInfo(); + + @BeforeEach + void setUp() { + + when(tesTaskService.cancelTesTask(STUBBED_TASK_ID)).thenReturn(new TesCancelTaskResponse()); + when(tesTaskService.listTesTask(NAME_PREFIX, PAGE_SIZE, PAGE_TOKEN, DEFAULT_VIEW)) + .thenReturn(mock(TesListTasksResponse.class)); + when(tesTaskService.getServiceInfo()).thenReturn(tesServiceInfo); + } + @Test void submitTesTaskWhenRequestingTesTaskBodyAndReturnId() throws Exception { - TesCreateTaskResponse tesCreateTaskResponse = new TesCreateTaskResponse(); tesCreateTaskResponse.setId(STUBBED_TASK_ID); when(tesTaskService.submitTesTask(any(TesTask.class))).thenReturn(tesCreateTaskResponse); - this.mockMvc.perform(post("/v1/tasks").contentType(MediaType.APPLICATION_JSON_UTF8) + this.mockMvc.perform(post("/v1/tasks").contentType("application/json") .content(STUBBED_SUBMIT_JSON_REQUEST)) .andDo(print()).andExpect(status().isOk()).andExpect(content().json(STUBBED_SUBMIT_JSON_RESPONSE)); } @@ -73,18 +81,16 @@ void expectIllegalArgExceptionWhenRunSubmitTesTasWithNullId() throws Exception { @Test void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { - when(tesTaskService.cancelTesTask(STUBBED_TASK_ID)).thenReturn(new TesCancelTaskResponse()); this.mockMvc.perform(post("/v1/tasks/{id}:cancel", STUBBED_TASK_ID)) .andDo(print()).andExpect(status().isOk()); } @Test void listTesTaskWhenRequestingReturnTesListTasksResponse() throws Exception { - when(tesTaskService.listTesTask(NAME_PREFIX, PAGE_SIZE, PAGE_TOKEN, DEFAULT_VIEW)) - .thenReturn(new TesListTasksResponse()); this.mockMvc.perform(get("/v1/tasks?name_prefix={name_prefix}?page_size={page_size}" + "?page_token={page_token}?view={view}", NAME_PREFIX, PAGE_SIZE, PAGE_TOKEN, DEFAULT_VIEW)) .andDo(print()).andExpect(status().isOk()); } + } From e42ccb1e691f444b2e9e5d8a77de3dc53df9a143 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 18 Sep 2019 17:48:42 +0300 Subject: [PATCH 113/194] added some tests --- .../tesadapter/service/TaskMapper.java | 6 +-- .../tesadapter/service/TaskMapperTest.java | 44 +++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index e579a85f73..60dc897490 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -242,7 +242,7 @@ private TesTask filterTesTaskWithView(PipelineRun run, TaskView view) { } } - private TesState createTesState(PipelineRun run) { + TesState createTesState(PipelineRun run) { List pipelineTaskList = cloudPipelineAPIClient.loadPipelineTasks(run.getId()); switch (run.getStatus()) { case RUNNING: @@ -268,7 +268,7 @@ private TesState createTesState(PipelineRun run) { } } - private List createTesInput(List parameters) { + List createTesInput(List parameters) { return parameters.stream() .filter(pipelineRunParameter -> pipelineRunParameter.getType().contains(INPUT_TYPE)) .map(pipelineRunParameter -> @@ -278,7 +278,7 @@ private List createTesInput(List parameters) { .build()).collect(Collectors.toList()); } - private List createTesOutput(List parameters) { + List createTesOutput(List parameters) { return parameters.stream() .filter(pipelineRunParameter -> pipelineRunParameter.getType().contains(OUTPUT_TYPE)) .map(pipelineRunParameter -> diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index a474b191c5..621b5e5fbb 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -2,6 +2,9 @@ import com.epam.pipeline.entity.cluster.AllowedInstanceAndPriceTypes; import com.epam.pipeline.entity.cluster.InstanceType; +import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.entity.pipeline.PipelineTask; +import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.entity.pipeline.Tool; import com.epam.pipeline.entity.region.AbstractCloudRegion; import com.epam.pipeline.tesadapter.common.MessageConstants; @@ -9,6 +12,7 @@ import com.epam.pipeline.tesadapter.configuration.AppConfiguration; import com.epam.pipeline.tesadapter.entity.TesExecutor; import com.epam.pipeline.tesadapter.entity.TesResources; +import com.epam.pipeline.tesadapter.entity.TesState; import com.epam.pipeline.tesadapter.entity.TesTask; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -22,10 +26,12 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; @@ -59,6 +65,7 @@ class TaskMapperTest { private AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = mock(AllowedInstanceAndPriceTypes.class); private Tool tool = mock(Tool.class); private TesTask tesTask = mock(TesTask.class); + private PipelineRun pipelineRun = mock(PipelineRun.class); @BeforeAll public static void setUpAll() { @@ -222,4 +229,41 @@ private static InstanceType getInstanceWithNameRamAndCpu(String instanceName, Do instanceType.setVCPU(instanceCpu.intValue()); return instanceType; } + + @Test + public void testCreateTesState() { + when(pipelineRun.getStatus()).thenReturn(TaskStatus.RUNNING); + assertEquals(TesState.RUNNING, taskMapper.createTesState(pipelineRun)); + + when(pipelineRun.getStatus()).thenReturn(TaskStatus.STOPPED); + assertEquals(TesState.CANCELED, taskMapper.createTesState(pipelineRun)); + + when(pipelineRun.getStatus()).thenReturn(TaskStatus.PAUSED); + assertEquals(TesState.PAUSED, taskMapper.createTesState(pipelineRun)); + + when(pipelineRun.getStatus()).thenReturn(TaskStatus.RUNNING); + List pipelineTaskList = Arrays.asList(mock(PipelineTask.class)); + when(cloudPipelineAPIClient.loadPipelineTasks(pipelineRun.getId())).thenReturn(pipelineTaskList); + when(pipelineTaskList.get(0).getName()).thenReturn("Console"); + assertEquals(TesState.QUEUED, taskMapper.createTesState(pipelineRun)); + + when(pipelineRun.getStatus()).thenReturn(TaskStatus.RUNNING); + when(pipelineTaskList.get(0).getName()).thenReturn("InitializeEnvironment"); + assertEquals(TesState.RUNNING, taskMapper.createTesState(pipelineRun)); + + when(pipelineRun.getStatus()).thenReturn(TaskStatus.RESUMING); + assertEquals(TesState.UNKNOWN, taskMapper.createTesState(pipelineRun)); + + assertNotNull(taskMapper.createTesState(pipelineRun)); + } + + @Test + public void testCreateTesInput() { + assertNotNull(taskMapper.createTesInput(pipelineRun.getPipelineRunParameters())); + } + + @Test + public void testCreateTesOutput(){ + assertNotNull(taskMapper.createTesOutput(pipelineRun.getPipelineRunParameters())); + } } \ No newline at end of file From a773eae12898023848db5638b22f3eb8cd4cd49f Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 18 Sep 2019 18:06:41 +0300 Subject: [PATCH 114/194] refactored listTesTask method for better readability --- .../tesadapter/service/TesTaskServiceImpl.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 953b5391f7..c858ac8206 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -69,11 +69,17 @@ public TesCreateTaskResponse submitTesTask(TesTask body) { @Override public TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String pageToken, TaskView view) { - return new TesListTasksResponse((StringUtils.isNotEmpty(namePrefix) ? - searchRunsWithNamePrefix(namePrefix, pageSize, pageToken) - : filterRunsWithOutNamePrefix(pageSize, pageToken)).stream().map(pipelineRun -> - taskMapper.mapToTesTask(pipelineRun, Optional.ofNullable(view).orElse(DEFAULT_TASK_VIEW))) - .collect(Collectors.toList())); + final List tesTaskList = (StringUtils.isNotEmpty(namePrefix) + ? searchRunsWithNamePrefix(namePrefix, pageSize, pageToken) + : filterRunsWithOutNamePrefix(pageSize, pageToken) + ).stream() + .map(pipelineRun -> + taskMapper.mapToTesTask( + pipelineRun, + Optional.ofNullable(view).orElse(DEFAULT_TASK_VIEW) + )) + .collect(Collectors.toList()); + return new TesListTasksResponse(tesTaskList); } private List searchRunsWithNamePrefix(String namePrefix, Long pageSize, String pageToken) { From 79e4d12b8c2d3069e1bdcc195459baec899a6510 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 18 Sep 2019 20:32:28 +0300 Subject: [PATCH 115/194] added some tests more --- .../tesadapter/service/TaskMapper.java | 2 +- .../tesadapter/service/TaskMapperTest.java | 64 ++++++++++++++++++- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 60dc897490..7201072953 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -302,7 +302,7 @@ private List getPipelineRunZone(PipelineRun run) { return Collections.singletonList(cloudPipelineAPIClient.loadRegion(run.getInstance().getCloudRegionId()).getRegionCode()); } - private List createListExecutor(PipelineRun run) { + List createListExecutor(PipelineRun run) { return ListUtils.emptyIfNull(Collections.singletonList(TesExecutor.builder() .command(ListUtils.emptyIfNull(Arrays.asList(run.getActualCmd().split(SEPARATOR)))) .env(run.getEnvVars()) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index 621b5e5fbb..c67ce6461d 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -4,8 +4,10 @@ import com.epam.pipeline.entity.cluster.InstanceType; import com.epam.pipeline.entity.pipeline.PipelineRun; import com.epam.pipeline.entity.pipeline.PipelineTask; +import com.epam.pipeline.entity.pipeline.RunInstance; import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.entity.pipeline.Tool; +import com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter; import com.epam.pipeline.entity.region.AbstractCloudRegion; import com.epam.pipeline.tesadapter.common.MessageConstants; import com.epam.pipeline.tesadapter.common.MessageHelper; @@ -27,6 +29,9 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.stream.Stream; @@ -260,10 +265,67 @@ public void testCreateTesState() { @Test public void testCreateTesInput() { assertNotNull(taskMapper.createTesInput(pipelineRun.getPipelineRunParameters())); + assertEquals("pipelineRunParameter", taskMapper.createTesInput(getListPipelineRunParameter()).get(0).getName()); + assertEquals("www.url.ru", taskMapper.createTesInput(getListPipelineRunParameter()).get(0).getUrl()); + + } + + private List getListPipelineRunParameter() { + PipelineRunParameter pipelineRunParameterInput = new PipelineRunParameter(); + pipelineRunParameterInput.setType("input"); + pipelineRunParameterInput.setName("pipelineRunParameter"); + pipelineRunParameterInput.setValue("www.url.ru"); + + PipelineRunParameter pipelineRunParameterOuput = new PipelineRunParameter(); + pipelineRunParameterOuput.setType("output"); + pipelineRunParameterOuput.setName("pipelineRunParameter"); + pipelineRunParameterOuput.setValue("www.url.ru"); + + List pipelineRunParameterList = new ArrayList<>(); + pipelineRunParameterList.add(pipelineRunParameterInput); + pipelineRunParameterList.add(pipelineRunParameterOuput); + return pipelineRunParameterList; } @Test - public void testCreateTesOutput(){ + public void testCreateTesOutput() { assertNotNull(taskMapper.createTesOutput(pipelineRun.getPipelineRunParameters())); + assertEquals("pipelineRunParameter", taskMapper.createTesOutput(getListPipelineRunParameter()).get(0).getName()); + assertEquals("www.url.ru", taskMapper.createTesOutput(getListPipelineRunParameter()).get(0).getUrl()); + } + + @Test + public void testCreateListExecutor() { + List listTesExecutor = new ArrayList<>(); + listTesExecutor.add(TesExecutor.builder() + .command(Collections.singletonList(getPipelineRun().getActualCmd())) + .env(getPipelineRun().getEnvVars()) + .image(getPipelineRun().getDockerImage()) + .build()); + assertEquals(listTesExecutor, taskMapper.createListExecutor(getPipelineRun())); + } + + private PipelineRun getPipelineRun() { + PipelineRun pipelineRun = new PipelineRun(); + pipelineRun.setId(1L); + pipelineRun.setStatus(TaskStatus.RUNNING); + pipelineRun.setPodIP("pipelineRun"); + pipelineRun.setPipelineRunParameters(getListPipelineRunParameter()); + pipelineRun.setStartDate(new Date(2000)); + pipelineRun.setInstance(getRunInstance()); + pipelineRun.setActualCmd("cmd"); + HashMap env = new HashMap<>(); + env.put("env", "env"); + pipelineRun.setEnvVars(env); + pipelineRun.setDockerImage("centos"); + return pipelineRun; + } + + private RunInstance getRunInstance() { + RunInstance runInstance = new RunInstance(); + runInstance.setSpot(true); + runInstance.setNodeDisk(1000); + runInstance.setCloudRegionId(1L); + return runInstance; } } \ No newline at end of file From 64efa445d786d1584d88f5a6388245eecd92b20d Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 19 Sep 2019 14:35:34 +0300 Subject: [PATCH 116/194] added more integration WebMvc tests to cover TesAdapterController class, refactored TesExceptionHandler, fixed message-outputs (changed name file from message.properties to messages.properties). Rise coverage for TesAdapterController to 100%. updated build.gradle to be able to use "jacoco". --- tes-adapter/build.gradle | 1 + .../controller/TesExceptionHandler.java | 14 +++- .../tesadapter/service/TaskMapper.java | 4 +- .../service/TesTaskServiceImpl.java | 2 +- ...message.properties => messages.properties} | 2 +- .../app/TesAdapterControllerTest.java | 75 +++++++++++++++++-- .../tesadapter/service/TaskMapperTest.java | 6 +- 7 files changed, 85 insertions(+), 19 deletions(-) rename tes-adapter/src/main/resources/{message.properties => messages.properties} (76%) diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index 3a30cf6ed6..59f22b1277 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -14,6 +14,7 @@ apply plugin: 'java' apply plugin: 'idea' apply plugin: 'org.springframework.boot' apply plugin: 'io.spring.dependency-management' +apply plugin: 'jacoco' repositories { mavenCentral() diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java index eba9e3a111..5f9867fdae 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java @@ -1,6 +1,8 @@ package com.epam.pipeline.tesadapter.controller; +import com.epam.pipeline.tesadapter.common.MessageHelper; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.ExceptionHandler; @@ -8,14 +10,18 @@ import org.springframework.web.context.request.WebRequest; @Slf4j -@RestControllerAdvice("com.epam.pipeline.tesadapter.controller") +@RestControllerAdvice() public class TesExceptionHandler { + @Autowired + private MessageHelper messageHelper; + @ExceptionHandler(Throwable.class) public final ResponseEntity handleUncaughtException(final Throwable exception, final WebRequest request) { - log.error(exception.getMessage() + request.getDescription(true)); - return new ResponseEntity<>(exception.getMessage() + request.getDescription(true), - HttpStatus.INTERNAL_SERVER_ERROR); + log.error(messageHelper.getMessage("logger.error", request.getDescription(true)), exception); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(exception.getMessage() + request.getDescription(true)); + } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index e579a85f73..a9d9dfbf33 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -127,7 +127,7 @@ public PipelineStart mapToPipelineStart(TesTask tesTask) { public TesExecutor getExecutorFromTesExecutorsList(List tesExecutors) { Assert.isTrue(tesExecutors.size() == ONLY_ONE, messageHelper.getMessage( - MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, EXECUTORS, tesExecutors)); + MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, EXECUTORS)); return tesExecutors.get(FIRST); } @@ -160,7 +160,7 @@ public Tool loadToolByTesImage(String image) { public Long getProperRegionIdInCloudRegionsByTesZone(List zones) { Assert.isTrue(zones.size() == ONLY_ONE, messageHelper.getMessage( - MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, ZONES, zones)); + MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, ZONES)); return Optional.ofNullable(cloudPipelineAPIClient.loadAllRegions().stream().filter( region -> region.getName().equalsIgnoreCase(zones.get(FIRST))) .collect(Collectors.toList()).get(FIRST).getId()).orElseThrow(() -> diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index c858ac8206..1a72d4041b 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -122,7 +122,7 @@ public TesTask getTesTask(String id, TaskView view) { private Long parseRunId(String id) { Assert.state(StringUtils.isNumeric(id), - messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, "ID", id)); + messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, ID)); return Long.parseLong(id); } diff --git a/tes-adapter/src/main/resources/message.properties b/tes-adapter/src/main/resources/messages.properties similarity index 76% rename from tes-adapter/src/main/resources/message.properties rename to tes-adapter/src/main/resources/messages.properties index a43b1aec2b..443bdc611b 100644 --- a/tes-adapter/src/main/resources/message.properties +++ b/tes-adapter/src/main/resources/messages.properties @@ -5,4 +5,4 @@ error.parameter.required=Parameter ''{0}'' id required for ''{1}'' podId. #Parameters error.parameter.non.scalar.type=Unable to resolve parameter value ''{0}''. Reference value for field ''{1}'' is used as a parameter value. -error.parameter.incompatible.content=Parameter (id ''{0}'') is not compatible with content ''{1}''. \ No newline at end of file +error.parameter.incompatible.content=Parameter (id ''{0}'') with undesirable content. \ No newline at end of file diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index eb91344414..a33f9d5e38 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -1,22 +1,31 @@ package com.epam.pipeline.tesadapter.app; +import com.epam.pipeline.tesadapter.common.MessageConstants; +import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.controller.TesAdapterController; import com.epam.pipeline.tesadapter.entity.TaskView; import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; import com.epam.pipeline.tesadapter.entity.TesCreateTaskResponse; +import com.epam.pipeline.tesadapter.entity.TesExecutor; import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; import com.epam.pipeline.tesadapter.entity.TesServiceInfo; import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.tesadapter.service.CloudPipelineAPIClient; import com.epam.pipeline.tesadapter.service.TesTaskServiceImpl; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.web.servlet.MockMvc; +import java.util.ArrayList; +import java.util.Collections; + +import static org.hamcrest.Matchers.containsString; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -30,13 +39,28 @@ @WebMvcTest(TesAdapterController.class) @SuppressWarnings({"unused", "PMD.TooManyStaticImports"}) public class TesAdapterControllerTest { - private static final String STUBBED_TASK_ID = "5"; + private static final String DEFAULT_TASK_ID = "5"; private static final String PAGE_TOKEN = "1"; private static final String NAME_PREFIX = "pipeline"; + private static final String EMPTY_INPUT = " "; + private static final String DEFAULT_COMMAND = "sleep 300"; + private static final String DEFAULT_IMAGE = + "cp-docker-registry.default.svc.cluster.local:31443/library/centos:latest"; private static final Long PAGE_SIZE = 20L; private static final TaskView DEFAULT_VIEW = TaskView.MINIMAL; private static final String STUBBED_SUBMIT_JSON_REQUEST = "{}"; private static final String STUBBED_SUBMIT_JSON_RESPONSE = "{\"id\":\"5\"}"; + private static final String CANCEL_REQUEST_DESCRIPTION = "uri=/v1/tasks/%20:cancel;client=127.0.0.1"; + private static final String JSON_CONTENT_TYPE = "application/json"; + + @Value("${cloud.pipeline.service.name}") + private String nameOfService; + + @Value("${cloud.pipeline.doc}") + private String doc; + + @Autowired + private MessageHelper messageHelper; @Autowired private MockMvc mockMvc; @@ -49,42 +73,58 @@ public class TesAdapterControllerTest { private TesCreateTaskResponse tesCreateTaskResponse = new TesCreateTaskResponse(); private TesServiceInfo tesServiceInfo = new TesServiceInfo(); + private TesTask tesTask = new TesTask(); + private TesExecutor tesExecutor = new TesExecutor(); @BeforeEach void setUp() { - when(tesTaskService.cancelTesTask(STUBBED_TASK_ID)).thenReturn(new TesCancelTaskResponse()); + when(tesTaskService.cancelTesTask(DEFAULT_TASK_ID)).thenReturn(new TesCancelTaskResponse()); when(tesTaskService.listTesTask(NAME_PREFIX, PAGE_SIZE, PAGE_TOKEN, DEFAULT_VIEW)) .thenReturn(mock(TesListTasksResponse.class)); when(tesTaskService.getServiceInfo()).thenReturn(tesServiceInfo); + when(tesTaskService.getTesTask(DEFAULT_TASK_ID, DEFAULT_VIEW)).thenReturn(tesTask); } @Test void submitTesTaskWhenRequestingTesTaskBodyAndReturnId() throws Exception { - tesCreateTaskResponse.setId(STUBBED_TASK_ID); + tesCreateTaskResponse.setId(DEFAULT_TASK_ID); when(tesTaskService.submitTesTask(any(TesTask.class))).thenReturn(tesCreateTaskResponse); - this.mockMvc.perform(post("/v1/tasks").contentType("application/json") + this.mockMvc.perform(post("/v1/tasks").contentType(JSON_CONTENT_TYPE) .content(STUBBED_SUBMIT_JSON_REQUEST)) - .andDo(print()).andExpect(status().isOk()).andExpect(content().json(STUBBED_SUBMIT_JSON_RESPONSE)); + .andDo(print()).andExpect(status().isOk()).andExpect(content() + .json(new ObjectMapper().writeValueAsString(tesCreateTaskResponse))); } @Test void expectIllegalArgExceptionWhenRunSubmitTesTasWithNullId() throws Exception { - TesTask tesTask = new TesTask(); tesTask.setExecutors(null); when(tesTaskService.submitTesTask(tesTask)).thenThrow(new IllegalArgumentException()); this.mockMvc.perform(post("/v1/tasks") - .contentType("application/json") + .contentType(JSON_CONTENT_TYPE) .content(STUBBED_SUBMIT_JSON_REQUEST)) .andDo(print()).andExpect(status().isInternalServerError()); } @Test void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { - this.mockMvc.perform(post("/v1/tasks/{id}:cancel", STUBBED_TASK_ID)) + when(tesTaskService.cancelTesTask(DEFAULT_TASK_ID)).thenReturn(new TesCancelTaskResponse()); + this.mockMvc.perform(post("/v1/tasks/{id}:cancel", DEFAULT_TASK_ID)) .andDo(print()).andExpect(status().isOk()); } + @Test + void expectIllegalStateExceptionWhenRunCancelTesTaskWithWrongId() throws Exception { + when(tesTaskService.cancelTesTask(EMPTY_INPUT)).thenThrow(new IllegalStateException(messageHelper + .getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, "taskId"))); + this.mockMvc.perform(post("/v1/tasks/{id}:cancel", EMPTY_INPUT)) + .andDo(print()) + .andExpect(status().isInternalServerError()) + .andExpect(content().string(containsString(messageHelper + .getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, "taskId") + + CANCEL_REQUEST_DESCRIPTION))); + } + @Test void listTesTaskWhenRequestingReturnTesListTasksResponse() throws Exception { this.mockMvc.perform(get("/v1/tasks?name_prefix={name_prefix}?page_size={page_size}" + @@ -93,4 +133,23 @@ void listTesTaskWhenRequestingReturnTesListTasksResponse() throws Exception { .andDo(print()).andExpect(status().isOk()); } + @Test + void getTesTaskWhenRequestingReturnTesTaskResponse() throws Exception { + tesExecutor.setImage(DEFAULT_IMAGE); + tesExecutor.setCommand(Collections.singletonList(DEFAULT_COMMAND)); + tesTask.setExecutors(Collections.singletonList(tesExecutor)); + this.mockMvc.perform(get("/v1/tasks/{id}", DEFAULT_TASK_ID).contentType(JSON_CONTENT_TYPE)) + .andDo(print()).andExpect(status().isOk()).andExpect(content() + .json(new ObjectMapper().writeValueAsString(tesTask))); + } + + @Test + void serviceInfoRequestShouldReturnCurrentServiceState() throws Exception { + tesServiceInfo.setName(nameOfService); + tesServiceInfo.setDoc(doc); + tesServiceInfo.setStorage(new ArrayList<>()); + this.mockMvc.perform(get("/v1/tasks/service-info").contentType(JSON_CONTENT_TYPE)) + .andDo(print()).andExpect(status().isOk()) + .andExpect(content().json(new ObjectMapper().writeValueAsString(tesServiceInfo))); + } } diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index a474b191c5..39cd45979f 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -91,7 +91,7 @@ public void expectIllegalArgExceptionWhenRunGetExecutorFromTesExecutorsList() { IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> taskMapper.getExecutorFromTesExecutorsList(tesExecutors)); assertTrue(exception.getMessage().contains(messageHelper.getMessage( - MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, EXECUTORS, tesExecutors))); + MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, EXECUTORS))); } @ParameterizedTest @@ -109,8 +109,8 @@ public void expectIllegalArgExceptionWhenRunGetProperRegionIdInCloudRegionsByTes zones.add("Second zone"); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> taskMapper.getProperRegionIdInCloudRegionsByTesZone(zones)); - assertTrue(exception.getMessage().contains(messageHelper.getMessage( - MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, EXECUTORS, zones))); + assertEquals(exception.getMessage(), messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, "zones")); } @Test From f3375b39a68aeeb34e1d1f2bf939b3423a34ab6d Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 19 Sep 2019 18:43:54 +0300 Subject: [PATCH 117/194] couple unit tests for TaskMapper class, increased coverage rate, refactored TaskMapper for better performance --- .../tesadapter/service/TaskMapper.java | 16 +++---- .../tesadapter/service/TaskMapperTest.java | 48 ++++++++++++++++--- 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index a9d9dfbf33..c8997d7aa2 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -57,6 +57,7 @@ public class TaskMapper { private static final String OUTPUT_TYPE = "output"; private static final String DEFAULT_TYPE = "string"; private static final String TOOL = "tool"; + private static final String COMMAND = "command"; private static final String INSTANCE_TYPES = "instanceList"; private static final String MIN_INSTANCE = "instance"; private static final String REGION_ID = "id"; @@ -100,16 +101,13 @@ public PipelineStart mapToPipelineStart(TesTask tesTask) { MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, IMAGE)); Tool pipelineTool = loadToolByTesImage(tesExecutor.getImage()); pipelineStart.setInstanceType(getProperInstanceType(tesTask, pipelineTool)); - pipelineStart.setCmdTemplate(String.join(SEPARATOR, tesExecutor.getCommand())); + pipelineStart.setCmdTemplate(String.join(SEPARATOR, Optional.ofNullable(tesExecutor.getCommand()) + .orElseThrow(() -> new IllegalArgumentException(messageHelper + .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, COMMAND))))); pipelineStart.setDockerImage(tesExecutor.getImage()); pipelineStart.setExecutionEnvironment(ExecutionEnvironment.CLOUD_PLATFORM); pipelineStart.setHddSize(Optional.ofNullable(tesTask.getResources()) - .map(tesResources -> { - if (Optional.ofNullable(tesResources.getDiskGb()).isPresent()) { - return tesResources.getDiskGb().intValue(); - } - return defaultHddSize; - }).orElse(defaultHddSize)); + .map((tesResources) -> tesResources.getDiskGb().intValue()).orElse(defaultHddSize)); pipelineStart.setIsSpot(Optional.ofNullable(tesTask.getResources()) .map(tesResources -> Optional.ofNullable(tesResources.getPreemptible()).orElse(defaultPreemptible)) .orElse(defaultPreemptible)); @@ -151,8 +149,6 @@ public String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { } public Tool loadToolByTesImage(String image) { - Assert.hasText(image, messageHelper.getMessage( - MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, image)); return Optional.ofNullable(cloudPipelineAPIClient.loadTool(image)).orElseThrow(() -> new IllegalArgumentException(messageHelper .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, TOOL))); @@ -197,7 +193,7 @@ private Double calculateInstanceCoef(InstanceType instanceType, Double ramGb, Lo - ramGb) / ramGb + Math.abs((double) (instanceType.getVCPU() - cpuCores)) / cpuCores; } - private Double convertMemoryUnitTypeToGiB(String memoryUnit) { + public Double convertMemoryUnitTypeToGiB(String memoryUnit) { if (memoryUnit != null) { if (memoryUnit.equalsIgnoreCase(PipelineDiskMemoryTypes.KIB.getValue())) { return KIB_TO_GIB; diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index 39cd45979f..11aec9cff4 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -22,6 +22,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.stream.Stream; @@ -83,6 +84,44 @@ public void setUp() { DEFAULT_PREEMPTIBLE, DEFAULT_REGION_NAME, cloudPipelineAPIClient, this.messageHelper); } + @Test + void mapToPipelineStartShouldConvertTesTaskToPipelineStart() { + PipelineStart pipelineStart = taskMapper.mapToPipelineStart(tesTask); + assertEquals(DEFAULT_HDD_SIZE, pipelineStart.getHddSize()); + assertEquals(STUBBED_IMAGE, pipelineStart.getDockerImage()); + assertTrue(hasText(pipelineStart.getInstanceType())); + assertEquals(DEFAULT_PREEMPTIBLE, pipelineStart.getIsSpot()); + assertNotNull(pipelineStart.getParams()); + } + + @Test + void expectIllegalArgExceptionWhenRunMapToPipelineStartWithNullTesTask() { + tesTask = null; + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> taskMapper.mapToPipelineStart(tesTask)); + assertTrue(exception.getMessage().contains(messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, tesTask))); + } + + @Test + void expectIllegalArgExceptionWhenRunMapToPipelineStartWithNullCommandExecutor() { + tesExecutor.setCommand(null); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> taskMapper.mapToPipelineStart(tesTask)); + assertEquals(exception.getMessage(), (messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, COMMAND))); + } + + @Test + void expectIllegalArgExceptionWhenRunMapToPipelineStartWithNullOrEmptyTesImage() { + tesExecutor.setImage(null); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> taskMapper.mapToPipelineStart(tesTask)); + assertEquals(exception.getMessage(), (messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, IMAGE))); + + } + @Test() public void expectIllegalArgExceptionWhenRunGetExecutorFromTesExecutorsList() { List tesExecutors = new ArrayList<>(); @@ -121,14 +160,9 @@ public void getProperRegionIdInCloudRegionsByTesZoneShouldReturnRegionId() { @Test public void expectIllegalArgExceptionWhenRunLoadToolByTesImageWithEmptyOrNullImage() { IllegalArgumentException exception1 = assertThrows(IllegalArgumentException.class, - () -> taskMapper.loadToolByTesImage("")); + () -> taskMapper.loadToolByTesImage(STUBBED_IMAGE)); assertTrue(exception1.getMessage().contains(messageHelper.getMessage( - MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, ""))); - - IllegalArgumentException exception2 = assertThrows(IllegalArgumentException.class, - () -> taskMapper.loadToolByTesImage(null)); - assertTrue(exception2.getMessage().contains(messageHelper.getMessage( - MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, "null"))); + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, "tool"))); } @Test From eb50e73b3b82e51fb77ce541d076b1fea0c5bcbd Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 19 Sep 2019 18:57:19 +0300 Subject: [PATCH 118/194] couple unit tests for TaskMapper class, added updates which was shalfed --- .../tesadapter/service/TaskMapperTest.java | 68 +++++++++++++++++-- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index 11aec9cff4..86e350cd9f 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -3,11 +3,14 @@ import com.epam.pipeline.entity.cluster.AllowedInstanceAndPriceTypes; import com.epam.pipeline.entity.cluster.InstanceType; import com.epam.pipeline.entity.pipeline.Tool; +import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.entity.region.AbstractCloudRegion; import com.epam.pipeline.tesadapter.common.MessageConstants; import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.configuration.AppConfiguration; import com.epam.pipeline.tesadapter.entity.TesExecutor; +import com.epam.pipeline.tesadapter.entity.TesInput; +import com.epam.pipeline.tesadapter.entity.TesOutput; import com.epam.pipeline.tesadapter.entity.TesResources; import com.epam.pipeline.tesadapter.entity.TesTask; import org.junit.jupiter.api.BeforeAll; @@ -27,15 +30,17 @@ import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.springframework.util.StringUtils.hasText; @ExtendWith(value = SpringExtension.class) @ContextConfiguration(classes = {AppConfiguration.class}) -@SuppressWarnings("unused") +@SuppressWarnings({"unused", "PMD.TooManyStaticImports"}) class TaskMapperTest { private static final Integer DEFAULT_HDD_SIZE = 50; private static final Double DEFAULT_RAM_GB = 8.0; @@ -43,16 +48,33 @@ class TaskMapperTest { private static final Boolean DEFAULT_PREEMPTIBLE = true; private static final String DEFAULT_REGION_NAME = "eu-central-1"; private static final String EXECUTORS = "executors"; + private static final String SIMPLE_NAME = "name"; + private static final String SIMPLE_URL = "somePath"; + private static final String IMAGE = "image"; + private static final String COMMAND = "command"; + private static final String DEFAULT_COMMAND = "sleep 300"; private static final Long STUBBED_REGION_ID = 1L; private static final Long STUBBED_TOOL_ID = 11584L; private static final String STUBBED_IMAGE = "cp-docker-registry.default.svc.cluster.local:31443/library/centos:latest"; private static List allowedInstanceTypes; + private static final Double KIB_TO_GIB = 0.00000095367432; + private static final Double MIB_TO_GIB = 0.0009765625; + private static final Double GIB_TO_GIB = 1.0; + private static final Double TIB_TO_GIB = 1024.0; + private static final Double PIB_TO_GIB = 1048576.0; + private static final Double EIB_TO_GIB = 1073741824.0; @Autowired private MessageHelper messageHelper; private TaskMapper taskMapper; private List zones = new ArrayList<>(); + private TesExecutor tesExecutor = new TesExecutor(); + private TesInput tesInput = mock(TesInput.class); + private TesOutput tesOutput = mock(TesOutput.class); + private List tesExecutors = new ArrayList<>(); + private List tesInputs = new ArrayList<>(); + private List tesOutputs = new ArrayList<>(); private List abstractCloudRegions = new ArrayList<>(); private AbstractCloudRegion abstractCloudRegion = mock(AbstractCloudRegion.class); @@ -75,11 +97,29 @@ public void setUp() { when(cloudPipelineAPIClient.loadTool(STUBBED_IMAGE)).thenReturn(tool); when(cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(STUBBED_TOOL_ID, STUBBED_REGION_ID, DEFAULT_PREEMPTIBLE)).thenReturn(allowedInstanceAndPriceTypes); + when(allowedInstanceAndPriceTypes.getAllowedInstanceTypes()).thenReturn(allowedInstanceTypes); when(tool.getId()).thenReturn(STUBBED_TOOL_ID); when(tesTask.getResources()).thenReturn(mock(TesResources.class)); - when(tesTask.getResources().getPreemptible()).thenReturn(true); + when(tesTask.getResources().getPreemptible()).thenReturn(DEFAULT_PREEMPTIBLE); + when(tesTask.getResources().getDiskGb()).thenReturn(DEFAULT_HDD_SIZE.doubleValue()); + when(tesTask.getResources().getRamGb()).thenReturn(DEFAULT_RAM_GB); + when(tesTask.getResources().getZones()).thenReturn(Collections.singletonList(DEFAULT_REGION_NAME)); + when(tesTask.getExecutors()).thenReturn(tesExecutors); + when(tesTask.getInputs()).thenReturn(tesInputs); + when(tesTask.getOutputs()).thenReturn(tesOutputs); + when(tesOutput.getName()).thenReturn(SIMPLE_NAME); + when(tesInput.getName()).thenReturn(SIMPLE_NAME); + when(tesInput.getUrl()).thenReturn(SIMPLE_URL); + when(tesOutput.getUrl()).thenReturn(SIMPLE_URL); zones.add(DEFAULT_REGION_NAME); + tesInputs.add(tesInput); + tesOutputs.add(tesOutput); + tesExecutor.setImage(STUBBED_IMAGE); + tesExecutor.setCommand(Collections.singletonList(DEFAULT_COMMAND)); + tesExecutor.setEnv(Collections.singletonMap(SIMPLE_NAME, SIMPLE_URL)); abstractCloudRegions.add(abstractCloudRegion); + tesExecutors.add(tesExecutor); + tesTask.setExecutors(tesExecutors); this.taskMapper = new TaskMapper(DEFAULT_HDD_SIZE, DEFAULT_RAM_GB, DEFAULT_CPU_CORES, DEFAULT_PREEMPTIBLE, DEFAULT_REGION_NAME, cloudPipelineAPIClient, this.messageHelper); } @@ -124,8 +164,6 @@ void expectIllegalArgExceptionWhenRunMapToPipelineStartWithNullOrEmptyTesImage() @Test() public void expectIllegalArgExceptionWhenRunGetExecutorFromTesExecutorsList() { - List tesExecutors = new ArrayList<>(); - tesExecutors.add(new TesExecutor()); tesExecutors.add(new TesExecutor()); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> taskMapper.getExecutorFromTesExecutorsList(tesExecutors)); @@ -133,6 +171,16 @@ public void expectIllegalArgExceptionWhenRunGetExecutorFromTesExecutorsList() { MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, EXECUTORS))); } + @Test + void convertMemoryUnitTypeToGiBShouldReturnCorrectConvertCoefficients() { + assertEquals(KIB_TO_GIB, taskMapper.convertMemoryUnitTypeToGiB("KiB")); + assertEquals(MIB_TO_GIB, taskMapper.convertMemoryUnitTypeToGiB("MiB")); + assertEquals(GIB_TO_GIB, taskMapper.convertMemoryUnitTypeToGiB("GiB")); + assertEquals(TIB_TO_GIB, taskMapper.convertMemoryUnitTypeToGiB("TiB")); + assertEquals(PIB_TO_GIB, taskMapper.convertMemoryUnitTypeToGiB("PiB")); + assertEquals(EIB_TO_GIB, taskMapper.convertMemoryUnitTypeToGiB("EiB")); + } + @ParameterizedTest @MethodSource("provideInputForGetProperInstanceTypeTest") public void getProperInstanceTypeShouldReturnProperInstanceName(String instanceTypeName, Double ramGb, Long cpuCore) { @@ -158,7 +206,17 @@ public void getProperRegionIdInCloudRegionsByTesZoneShouldReturnRegionId() { } @Test - public void expectIllegalArgExceptionWhenRunLoadToolByTesImageWithEmptyOrNullImage() { + void expectIllegalArgExceptionWhenRunGetProperRegionIdWithWrongResponse() { + when(abstractCloudRegion.getId()).thenReturn(null); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> taskMapper.getProperRegionIdInCloudRegionsByTesZone(zones)); + assertEquals(exception.getMessage(), messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, "id")); + } + + @Test + public void expectIllegalArgExceptionWhenRunLoadToolByTesImageWithEmptyOrNullResponse() { + when(cloudPipelineAPIClient.loadTool(STUBBED_IMAGE)).thenReturn(null); IllegalArgumentException exception1 = assertThrows(IllegalArgumentException.class, () -> taskMapper.loadToolByTesImage(STUBBED_IMAGE)); assertTrue(exception1.getMessage().contains(messageHelper.getMessage( From abf6b8d2ef3c92e08b3a9a13eb94fb44b601ed7b Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Thu, 19 Sep 2019 22:10:08 +0300 Subject: [PATCH 119/194] added some tests more --- tes-adapter/build.gradle | 1 + .../tesadapter/service/TaskMapper.java | 14 +-- .../tesadapter/service/TaskMapperTest.java | 108 ++++++++++++++++-- 3 files changed, 107 insertions(+), 16 deletions(-) diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index 3a30cf6ed6..59f22b1277 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -14,6 +14,7 @@ apply plugin: 'java' apply plugin: 'idea' apply plugin: 'org.springframework.boot' apply plugin: 'io.spring.dependency-management' +apply plugin: 'jacoco' repositories { mavenCentral() diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 7201072953..8169a2571a 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -197,7 +197,7 @@ private Double calculateInstanceCoef(InstanceType instanceType, Double ramGb, Lo - ramGb) / ramGb + Math.abs((double) (instanceType.getVCPU() - cpuCores)) / cpuCores; } - private Double convertMemoryUnitTypeToGiB(String memoryUnit) { + Double convertMemoryUnitTypeToGiB(String memoryUnit) { if (memoryUnit != null) { if (memoryUnit.equalsIgnoreCase(PipelineDiskMemoryTypes.KIB.getValue())) { return KIB_TO_GIB; @@ -288,7 +288,7 @@ List createTesOutput(List parameters) { .build()).collect(Collectors.toList()); } - private TesResources createTesResources(PipelineRun run) { + TesResources createTesResources(PipelineRun run) { return TesResources.builder() .preemptible(run.getInstance().getSpot()) .diskGb(new Double(run.getInstance().getNodeDisk())) @@ -298,7 +298,7 @@ private TesResources createTesResources(PipelineRun run) { .build(); } - private List getPipelineRunZone(PipelineRun run) { + List getPipelineRunZone(PipelineRun run) { return Collections.singletonList(cloudPipelineAPIClient.loadRegion(run.getInstance().getCloudRegionId()).getRegionCode()); } @@ -310,7 +310,7 @@ List createListExecutor(PipelineRun run) { .build())); } - private List createTesTaskLog(final Long runId) { + List createTesTaskLog(final Long runId) { List runLogList = ListUtils.emptyIfNull(cloudPipelineAPIClient.getRunLog(runId)); List tesExecutorLogList = Collections.singletonList(TesExecutorLog.builder() .stdout(runLogList.stream() @@ -322,7 +322,7 @@ private List createTesTaskLog(final Long runId) { .build()); } - private InstanceType getInstanceType(PipelineRun run) { + InstanceType getInstanceType(PipelineRun run) { Long regionId = run.getInstance().getCloudRegionId(); Boolean spot = run.getInstance().getSpot(); Tool pipeLineTool = loadToolByTesImage(run.getDockerImage()); @@ -331,9 +331,7 @@ private InstanceType getInstanceType(PipelineRun run) { cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(pipeLineTool.getId(), regionId, spot); return Stream.of(allowedInstanceAndPriceTypes) .flatMap(instance -> instance.getAllowedInstanceTypes().stream()) - .filter(instance -> instance - .getName() - .equalsIgnoreCase(nodeType)) + .filter(instance -> instance.getName().equalsIgnoreCase(nodeType)) .findFirst().orElseThrow(() -> new IllegalArgumentException(messageHelper .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY))); } diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index c67ce6461d..8d8207ab9d 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -5,6 +5,7 @@ import com.epam.pipeline.entity.pipeline.PipelineRun; import com.epam.pipeline.entity.pipeline.PipelineTask; import com.epam.pipeline.entity.pipeline.RunInstance; +import com.epam.pipeline.entity.pipeline.RunLog; import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.entity.pipeline.Tool; import com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter; @@ -13,9 +14,11 @@ import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.configuration.AppConfiguration; import com.epam.pipeline.tesadapter.entity.TesExecutor; +import com.epam.pipeline.tesadapter.entity.TesExecutorLog; import com.epam.pipeline.tesadapter.entity.TesResources; import com.epam.pipeline.tesadapter.entity.TesState; import com.epam.pipeline.tesadapter.entity.TesTask; +import com.epam.pipeline.tesadapter.entity.TesTaskLog; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -39,6 +42,10 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -246,22 +253,54 @@ public void testCreateTesState() { when(pipelineRun.getStatus()).thenReturn(TaskStatus.PAUSED); assertEquals(TesState.PAUSED, taskMapper.createTesState(pipelineRun)); - when(pipelineRun.getStatus()).thenReturn(TaskStatus.RUNNING); - List pipelineTaskList = Arrays.asList(mock(PipelineTask.class)); - when(cloudPipelineAPIClient.loadPipelineTasks(pipelineRun.getId())).thenReturn(pipelineTaskList); - when(pipelineTaskList.get(0).getName()).thenReturn("Console"); - assertEquals(TesState.QUEUED, taskMapper.createTesState(pipelineRun)); + List pipelineTaskListWithConsole = getPipelineTaskListWithConsole(); + when(cloudPipelineAPIClient.loadPipelineTasks(anyLong())).thenReturn(pipelineTaskListWithConsole); + assertEquals(TesState.QUEUED, taskMapper.createTesState(getPipelineRun())); - when(pipelineRun.getStatus()).thenReturn(TaskStatus.RUNNING); - when(pipelineTaskList.get(0).getName()).thenReturn("InitializeEnvironment"); - assertEquals(TesState.RUNNING, taskMapper.createTesState(pipelineRun)); + List pipelineTaskListWithEnv = getPipelineTaskListWithEnv(); + when(cloudPipelineAPIClient.loadPipelineTasks(anyLong())).thenReturn(pipelineTaskListWithEnv); + assertEquals(TesState.RUNNING, taskMapper.createTesState(getPipelineRun())); + + List pipelineTaskListWithoutEnv = getPipelineTaskListWithoutEnv(); + when(cloudPipelineAPIClient.loadPipelineTasks(anyLong())).thenReturn(pipelineTaskListWithoutEnv); + assertEquals(TesState.INITIALIZING, taskMapper.createTesState(getPipelineRun())); when(pipelineRun.getStatus()).thenReturn(TaskStatus.RESUMING); assertEquals(TesState.UNKNOWN, taskMapper.createTesState(pipelineRun)); + when(pipelineRun.getStatus()).thenReturn(TaskStatus.SUCCESS); + assertEquals(TesState.COMPLETE, taskMapper.createTesState(pipelineRun)); + + when(pipelineRun.getStatus()).thenReturn(TaskStatus.FAILURE); + assertEquals(TesState.EXECUTOR_ERROR, taskMapper.createTesState(pipelineRun)); + assertNotNull(taskMapper.createTesState(pipelineRun)); } + private List getPipelineTaskListWithConsole(){ + ArrayList pipelineTasksList = new ArrayList<>(); + PipelineTask pipelineTaskConsole = new PipelineTask(); + pipelineTaskConsole.setName("Console"); + pipelineTasksList.add(pipelineTaskConsole); + return pipelineTasksList; + } + + private List getPipelineTaskListWithEnv(){ + ArrayList pipelineTasksList = new ArrayList<>(); + PipelineTask pipelineTaskInitializeEnvironment = new PipelineTask(); + pipelineTaskInitializeEnvironment.setName("InitializeEnvironment"); + pipelineTasksList.add(pipelineTaskInitializeEnvironment); + return pipelineTasksList; + } + + private List getPipelineTaskListWithoutEnv(){ + List pipelineTasksList = getPipelineTaskListWithConsole(); + PipelineTask pipelineTask = new PipelineTask(); + pipelineTask.setName("someState"); + pipelineTasksList.add(pipelineTask); + return pipelineTasksList; + } + @Test public void testCreateTesInput() { assertNotNull(taskMapper.createTesInput(pipelineRun.getPipelineRunParameters())); @@ -305,6 +344,23 @@ public void testCreateListExecutor() { assertEquals(listTesExecutor, taskMapper.createListExecutor(getPipelineRun())); } + @Test + public void testCreateTesTaskLog(){ + RunLog runLog = RunLog.builder() + .date(new Date(12,12,12)) + .taskName("log") + .logText("log") + .build(); + when(cloudPipelineAPIClient.getRunLog(anyLong())).thenReturn(Collections.singletonList(runLog)); + List tesExecutorLogList = Collections.singletonList(TesExecutorLog.builder() + .stdout(String.format("%s - %s - %s", runLog.getDate(), runLog.getTaskName(), runLog.getLogText())) + .build()); + List tesTaskLogsList = Collections.singletonList(TesTaskLog.builder() + .logs(tesExecutorLogList) + .build()); + assertEquals(tesTaskLogsList, taskMapper.createTesTaskLog(anyLong())); + } + private PipelineRun getPipelineRun() { PipelineRun pipelineRun = new PipelineRun(); pipelineRun.setId(1L); @@ -323,9 +379,45 @@ private PipelineRun getPipelineRun() { private RunInstance getRunInstance() { RunInstance runInstance = new RunInstance(); + runInstance.setNodeType("nodeType"); runInstance.setSpot(true); runInstance.setNodeDisk(1000); runInstance.setCloudRegionId(1L); return runInstance; } + + private InstanceType getInstanceType(){ + return InstanceType.builder() + .name(getPipelineRun().getInstance().getNodeType()) + .memory(10) + .memoryUnit("KiB") + .vCPU(8) + .build(); + } + + @Test + public void testGetInstanceType(){ + AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = + new AllowedInstanceAndPriceTypes(Collections.singletonList(getInstanceType()), + Collections.singletonList(getInstanceType()), Collections.singletonList("string")); + when(cloudPipelineAPIClient.loadTool(anyString())).thenReturn(mock(Tool.class)); + when(cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(anyLong(), anyLong(), anyBoolean())).thenReturn(allowedInstanceAndPriceTypes); + InstanceType instanceType = getInstanceType(); + assertEquals(instanceType.getName(), taskMapper.getInstanceType(getPipelineRun()).getName()); + assertEquals(instanceType.getMemory(), taskMapper.getInstanceType(getPipelineRun()).getMemory()); + assertEquals(instanceType.getMemoryUnit(), taskMapper.getInstanceType(getPipelineRun()).getMemoryUnit()); + assertEquals(instanceType.getVCPU(), taskMapper.getInstanceType(getPipelineRun()).getVCPU()); + } + +// @Test +// public void testCreateTesResources(){ +// TesResources tesResources = TesResources.builder() +// .preemptible(getPipelineRun().getInstance().getSpot()) +// .diskGb(new Double(getPipelineRun().getInstance().getNodeDisk())) +// .ramGb(getInstanceType().getMemory() * taskMapper.convertMemoryUnitTypeToGiB(getInstanceType().getMemoryUnit())) +// .cpuCores((long) getInstanceType().getVCPU()) +// .zones(Collections.singletonList(DEFAULT_REGION_NAME)) +// .build(); +// assertEquals(tesResources, taskMapper.createTesResources(getPipelineRun())); +// } } \ No newline at end of file From f9649821b3d18845910548bbf4c1d349f4de31aa Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 19 Sep 2019 22:13:31 +0300 Subject: [PATCH 120/194] added unit-tests for TaskMapper, 58% coverage --- .../tesadapter/service/TaskMapper.java | 2 +- .../tesadapter/service/TaskMapperTest.java | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index c8997d7aa2..018815d9f3 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -164,7 +164,7 @@ public Long getProperRegionIdInCloudRegionsByTesZone(List zones) { .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, REGION_ID))); } - private String evaluateMostProperInstanceType(AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes, + public String evaluateMostProperInstanceType(AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes, Double ramGb, Long cpuCores) { return Optional.ofNullable(allowedInstanceAndPriceTypes.getAllowedInstanceTypes()) .orElseThrow(() -> diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index 86e350cd9f..a450bb496e 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -52,6 +52,8 @@ class TaskMapperTest { private static final String SIMPLE_URL = "somePath"; private static final String IMAGE = "image"; private static final String COMMAND = "command"; + private static final String INSTANCE_TYPES = "instanceList"; + private static final String MIN_INSTANCE = "instance"; private static final String DEFAULT_COMMAND = "sleep 300"; private static final Long STUBBED_REGION_ID = 1L; private static final Long STUBBED_TOOL_ID = 11584L; @@ -171,6 +173,28 @@ public void expectIllegalArgExceptionWhenRunGetExecutorFromTesExecutorsList() { MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, EXECUTORS))); } + @Test + void expectIllegalArgExceptionWhenRunEvaluateMostProperInstanceTypeWithNullInstanceTypesList() { + when(allowedInstanceAndPriceTypes.getAllowedInstanceTypes()).thenReturn(null); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> taskMapper.evaluateMostProperInstanceType(allowedInstanceAndPriceTypes, + DEFAULT_RAM_GB, DEFAULT_CPU_CORES)); + assertTrue(exception.getMessage().contains(messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, INSTANCE_TYPES))); + + } + + @Test + void expectIllegalArgExceptionWhenRunEvaluateMostProperInstanceTypeWithNullResponse() { + when(allowedInstanceAndPriceTypes.getAllowedInstanceTypes()).thenReturn(new ArrayList<>()); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> taskMapper.evaluateMostProperInstanceType(allowedInstanceAndPriceTypes, + DEFAULT_RAM_GB, DEFAULT_CPU_CORES)); + assertTrue(exception.getMessage().contains(messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, MIN_INSTANCE))); + + } + @Test void convertMemoryUnitTypeToGiBShouldReturnCorrectConvertCoefficients() { assertEquals(KIB_TO_GIB, taskMapper.convertMemoryUnitTypeToGiB("KiB")); From 3ac2e924a78d6589c3b8b4d2627e9c9f4995750b Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 19 Sep 2019 22:39:12 +0300 Subject: [PATCH 121/194] fixed bug with mapping "disk_gb" field in TaskMapper --- .../java/com/epam/pipeline/tesadapter/service/TaskMapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 018815d9f3..d6c3db580c 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -107,7 +107,7 @@ public PipelineStart mapToPipelineStart(TesTask tesTask) { pipelineStart.setDockerImage(tesExecutor.getImage()); pipelineStart.setExecutionEnvironment(ExecutionEnvironment.CLOUD_PLATFORM); pipelineStart.setHddSize(Optional.ofNullable(tesTask.getResources()) - .map((tesResources) -> tesResources.getDiskGb().intValue()).orElse(defaultHddSize)); + .map(TesResources::getDiskGb).map(Double::intValue).orElse(defaultHddSize)); pipelineStart.setIsSpot(Optional.ofNullable(tesTask.getResources()) .map(tesResources -> Optional.ofNullable(tesResources.getPreemptible()).orElse(defaultPreemptible)) .orElse(defaultPreemptible)); @@ -165,7 +165,7 @@ public Long getProperRegionIdInCloudRegionsByTesZone(List zones) { } public String evaluateMostProperInstanceType(AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes, - Double ramGb, Long cpuCores) { + Double ramGb, Long cpuCores) { return Optional.ofNullable(allowedInstanceAndPriceTypes.getAllowedInstanceTypes()) .orElseThrow(() -> new IllegalArgumentException(messageHelper From 946f1645c03b8685afbb250b7a99009a01170ab7 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 20 Sep 2019 13:56:59 +0300 Subject: [PATCH 122/194] increased encapsulation level of TaskMapper class - changed method-modifiers --- .../tesadapter/service/TaskMapper.java | 24 +-- .../tesadapter/service/TaskMapperTest.java | 174 +++++++++--------- 2 files changed, 97 insertions(+), 101 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index d6c3db580c..371907f48f 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -91,7 +91,7 @@ public TaskMapper(@Value("${cloud.pipeline.hddSize}") Integer hddSize, this.defaultRegion = defaultRegion; } - public PipelineStart mapToPipelineStart(TesTask tesTask) { + PipelineStart mapToPipelineStart(TesTask tesTask) { Assert.notNull(tesTask, messageHelper.getMessage( MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, tesTask)); PipelineStart pipelineStart = new PipelineStart(); @@ -123,14 +123,14 @@ public PipelineStart mapToPipelineStart(TesTask tesTask) { return pipelineStart; } - public TesExecutor getExecutorFromTesExecutorsList(List tesExecutors) { + private TesExecutor getExecutorFromTesExecutorsList(List tesExecutors) { Assert.isTrue(tesExecutors.size() == ONLY_ONE, messageHelper.getMessage( MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, EXECUTORS)); return tesExecutors.get(FIRST); } - public String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { + String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { Double ramGb = Optional.ofNullable(tesTask.getResources()) .map(TesResources::getRamGb).orElse(defaultRamGb); Long cpuCores = Optional.ofNullable(tesTask.getResources()) @@ -141,20 +141,20 @@ public String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { .orElse(Collections.singletonList(defaultRegion))); Boolean spot = Optional.ofNullable(tesTask.getResources()) .map(TesResources::getPreemptible).orElse(defaultPreemptible); - AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = cloudPipelineAPIClient - .loadAllowedInstanceAndPriceTypes(toolId, regionId, spot); - Assert.notEmpty(allowedInstanceAndPriceTypes.getAllowedInstanceTypes(), messageHelper.getMessage( - MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, allowedInstanceAndPriceTypes)); + AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = Optional.ofNullable(cloudPipelineAPIClient + .loadAllowedInstanceAndPriceTypes(toolId, regionId, spot)) + .orElseThrow(() -> new IllegalArgumentException(messageHelper + .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, INSTANCE_TYPES))); return evaluateMostProperInstanceType(allowedInstanceAndPriceTypes, ramGb, cpuCores); } - public Tool loadToolByTesImage(String image) { + private Tool loadToolByTesImage(String image) { return Optional.ofNullable(cloudPipelineAPIClient.loadTool(image)).orElseThrow(() -> new IllegalArgumentException(messageHelper .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, TOOL))); } - public Long getProperRegionIdInCloudRegionsByTesZone(List zones) { + private Long getProperRegionIdInCloudRegionsByTesZone(List zones) { Assert.isTrue(zones.size() == ONLY_ONE, messageHelper.getMessage( MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, ZONES)); return Optional.ofNullable(cloudPipelineAPIClient.loadAllRegions().stream().filter( @@ -164,8 +164,8 @@ public Long getProperRegionIdInCloudRegionsByTesZone(List zones) { .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, REGION_ID))); } - public String evaluateMostProperInstanceType(AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes, - Double ramGb, Long cpuCores) { + private String evaluateMostProperInstanceType(AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes, + Double ramGb, Long cpuCores) { return Optional.ofNullable(allowedInstanceAndPriceTypes.getAllowedInstanceTypes()) .orElseThrow(() -> new IllegalArgumentException(messageHelper @@ -193,7 +193,7 @@ private Double calculateInstanceCoef(InstanceType instanceType, Double ramGb, Lo - ramGb) / ramGb + Math.abs((double) (instanceType.getVCPU() - cpuCores)) / cpuCores; } - public Double convertMemoryUnitTypeToGiB(String memoryUnit) { + private Double convertMemoryUnitTypeToGiB(String memoryUnit) { if (memoryUnit != null) { if (memoryUnit.equalsIgnoreCase(PipelineDiskMemoryTypes.KIB.getValue())) { return KIB_TO_GIB; diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index a450bb496e..3a3ace3cbd 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -33,6 +33,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.springframework.util.StringUtils.hasText; @@ -51,6 +52,7 @@ class TaskMapperTest { private static final String SIMPLE_NAME = "name"; private static final String SIMPLE_URL = "somePath"; private static final String IMAGE = "image"; + private static final String TOOL = "tool"; private static final String COMMAND = "command"; private static final String INSTANCE_TYPES = "instanceList"; private static final String MIN_INSTANCE = "instance"; @@ -59,6 +61,7 @@ class TaskMapperTest { private static final Long STUBBED_TOOL_ID = 11584L; private static final String STUBBED_IMAGE = "cp-docker-registry.default.svc.cluster.local:31443/library/centos:latest"; private static List allowedInstanceTypes; + private static final String GIB = "GiB"; private static final Double KIB_TO_GIB = 0.00000095367432; private static final Double MIB_TO_GIB = 0.0009765625; private static final Double GIB_TO_GIB = 1.0; @@ -105,7 +108,7 @@ public void setUp() { when(tesTask.getResources().getPreemptible()).thenReturn(DEFAULT_PREEMPTIBLE); when(tesTask.getResources().getDiskGb()).thenReturn(DEFAULT_HDD_SIZE.doubleValue()); when(tesTask.getResources().getRamGb()).thenReturn(DEFAULT_RAM_GB); - when(tesTask.getResources().getZones()).thenReturn(Collections.singletonList(DEFAULT_REGION_NAME)); + when(tesTask.getResources().getZones()).thenReturn(zones); when(tesTask.getExecutors()).thenReturn(tesExecutors); when(tesTask.getInputs()).thenReturn(tesInputs); when(tesTask.getOutputs()).thenReturn(tesOutputs); @@ -127,7 +130,7 @@ public void setUp() { } @Test - void mapToPipelineStartShouldConvertTesTaskToPipelineStart() { + public void mapToPipelineStartShouldConvertTesTaskToPipelineStart() { PipelineStart pipelineStart = taskMapper.mapToPipelineStart(tesTask); assertEquals(DEFAULT_HDD_SIZE, pipelineStart.getHddSize()); assertEquals(STUBBED_IMAGE, pipelineStart.getDockerImage()); @@ -137,7 +140,7 @@ void mapToPipelineStartShouldConvertTesTaskToPipelineStart() { } @Test - void expectIllegalArgExceptionWhenRunMapToPipelineStartWithNullTesTask() { + public void expectIllegalArgExceptionWhenRunMapToPipelineStartWithNullTesTask() { tesTask = null; IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> taskMapper.mapToPipelineStart(tesTask)); @@ -146,7 +149,7 @@ void expectIllegalArgExceptionWhenRunMapToPipelineStartWithNullTesTask() { } @Test - void expectIllegalArgExceptionWhenRunMapToPipelineStartWithNullCommandExecutor() { + public void expectIllegalArgExceptionWhenRunMapToPipelineStartWithNullCommandExecutor() { tesExecutor.setCommand(null); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> taskMapper.mapToPipelineStart(tesTask)); @@ -155,54 +158,64 @@ void expectIllegalArgExceptionWhenRunMapToPipelineStartWithNullCommandExecutor() } @Test - void expectIllegalArgExceptionWhenRunMapToPipelineStartWithNullOrEmptyTesImage() { + public void expectIllegalArgExceptionWhenRunMapToPipelineStartWithNullOrEmptyTesImage() { tesExecutor.setImage(null); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> taskMapper.mapToPipelineStart(tesTask)); assertEquals(exception.getMessage(), (messageHelper.getMessage( MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, IMAGE))); + } + @Test + public void expectIllegalArgExceptionWhenRunMapToPipelineStartWithNullLoadedInstancesResponse() { + when(cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(STUBBED_TOOL_ID, + STUBBED_REGION_ID, DEFAULT_PREEMPTIBLE)).thenReturn(null); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> taskMapper.mapToPipelineStart(tesTask)); + assertEquals(exception.getMessage(), (messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, INSTANCE_TYPES))); } + @Test + public void expectIllegalArgExceptionWhenRunMapToPipelineStartWithNullTool() { + when(cloudPipelineAPIClient.loadTool(any())).thenReturn(null); + IllegalArgumentException exception2 = assertThrows(IllegalArgumentException.class, + () -> taskMapper.mapToPipelineStart(tesTask)); + assertEquals(exception2.getMessage(), (messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, TOOL))); + } + + @Test() public void expectIllegalArgExceptionWhenRunGetExecutorFromTesExecutorsList() { tesExecutors.add(new TesExecutor()); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, - () -> taskMapper.getExecutorFromTesExecutorsList(tesExecutors)); + () -> taskMapper.mapToPipelineStart(tesTask)); assertTrue(exception.getMessage().contains(messageHelper.getMessage( MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, EXECUTORS))); } @Test - void expectIllegalArgExceptionWhenRunEvaluateMostProperInstanceTypeWithNullInstanceTypesList() { + public void expectIllegalArgExceptionWhenRunGetProperInstanceTypeWithNullInstanceTypesList() { when(allowedInstanceAndPriceTypes.getAllowedInstanceTypes()).thenReturn(null); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, - () -> taskMapper.evaluateMostProperInstanceType(allowedInstanceAndPriceTypes, - DEFAULT_RAM_GB, DEFAULT_CPU_CORES)); - assertTrue(exception.getMessage().contains(messageHelper.getMessage( - MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, INSTANCE_TYPES))); - + () -> taskMapper.getProperInstanceType(tesTask, tool)); + assertEquals(exception.getMessage(), messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, INSTANCE_TYPES)); } @Test - void expectIllegalArgExceptionWhenRunEvaluateMostProperInstanceTypeWithNullResponse() { + public void expectIllegalArgExceptionWhenRunGetProperInstanceTypeWithNullMinInstanceResponse() { when(allowedInstanceAndPriceTypes.getAllowedInstanceTypes()).thenReturn(new ArrayList<>()); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, - () -> taskMapper.evaluateMostProperInstanceType(allowedInstanceAndPriceTypes, - DEFAULT_RAM_GB, DEFAULT_CPU_CORES)); - assertTrue(exception.getMessage().contains(messageHelper.getMessage( - MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, MIN_INSTANCE))); - + () -> taskMapper.getProperInstanceType(tesTask, tool)); + assertEquals(exception.getMessage(), messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, MIN_INSTANCE)); } @Test - void convertMemoryUnitTypeToGiBShouldReturnCorrectConvertCoefficients() { - assertEquals(KIB_TO_GIB, taskMapper.convertMemoryUnitTypeToGiB("KiB")); - assertEquals(MIB_TO_GIB, taskMapper.convertMemoryUnitTypeToGiB("MiB")); - assertEquals(GIB_TO_GIB, taskMapper.convertMemoryUnitTypeToGiB("GiB")); - assertEquals(TIB_TO_GIB, taskMapper.convertMemoryUnitTypeToGiB("TiB")); - assertEquals(PIB_TO_GIB, taskMapper.convertMemoryUnitTypeToGiB("PiB")); - assertEquals(EIB_TO_GIB, taskMapper.convertMemoryUnitTypeToGiB("EiB")); + void getProperInstanceTypeMethodShouldUseProperConversionCoefficientForInstanceMemoryUnit() { + } @ParameterizedTest @@ -210,48 +223,27 @@ void convertMemoryUnitTypeToGiBShouldReturnCorrectConvertCoefficients() { public void getProperInstanceTypeShouldReturnProperInstanceName(String instanceTypeName, Double ramGb, Long cpuCore) { when(tesTask.getResources().getRamGb()).thenReturn(ramGb); when(tesTask.getResources().getCpuCores()).thenReturn(cpuCore); - when(tesTask.getResources().getZones()).thenReturn(zones); - when(allowedInstanceAndPriceTypes.getAllowedInstanceTypes()).thenReturn(allowedInstanceTypes); assertEquals(instanceTypeName, taskMapper.getProperInstanceType(tesTask, tool)); } @Test - public void expectIllegalArgExceptionWhenRunGetProperRegionIdInCloudRegionsByTesZone() { + public void expectIllegalArgExceptionWhenRunGetProperInstanceTypeWithTwoZones() { zones.add("Second zone"); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, - () -> taskMapper.getProperRegionIdInCloudRegionsByTesZone(zones)); + () -> taskMapper.getProperInstanceType(tesTask, tool)); assertEquals(exception.getMessage(), messageHelper.getMessage( MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, "zones")); } @Test - public void getProperRegionIdInCloudRegionsByTesZoneShouldReturnRegionId() { - assertEquals(STUBBED_REGION_ID, taskMapper.getProperRegionIdInCloudRegionsByTesZone(zones)); - } - - @Test - void expectIllegalArgExceptionWhenRunGetProperRegionIdWithWrongResponse() { + public void expectIllegalArgExceptionWhenRunGetProperInstanceTypeWithWrongRegionIdResponse() { when(abstractCloudRegion.getId()).thenReturn(null); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, - () -> taskMapper.getProperRegionIdInCloudRegionsByTesZone(zones)); + () -> taskMapper.getProperInstanceType(tesTask, tool)); assertEquals(exception.getMessage(), messageHelper.getMessage( MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, "id")); } - @Test - public void expectIllegalArgExceptionWhenRunLoadToolByTesImageWithEmptyOrNullResponse() { - when(cloudPipelineAPIClient.loadTool(STUBBED_IMAGE)).thenReturn(null); - IllegalArgumentException exception1 = assertThrows(IllegalArgumentException.class, - () -> taskMapper.loadToolByTesImage(STUBBED_IMAGE)); - assertTrue(exception1.getMessage().contains(messageHelper.getMessage( - MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, "tool"))); - } - - @Test - public void loadToolByTesImageShouldReturnToolByRequestingImage() { - assertEquals(tool, taskMapper.loadToolByTesImage(STUBBED_IMAGE)); - } - private static Stream provideInputForGetProperInstanceTypeTest() { return Stream.of( //Perfect cases with 0.0 deviation @@ -276,66 +268,70 @@ private static Stream provideInputForGetProperInstanceTypeTest() { Arguments.of("p2.16xlarge", 768.0, 64L), Arguments.of("p2.8xlarge", 488.0, 32L), Arguments.of("p2.xlarge", 61.0, 4L), - Arguments.of("r5.12xlarge", 384.0, 48L), - Arguments.of("r5.16xlarge", 512.0, 64L), - Arguments.of("r5.24xlarge", 768.0, 96L), - Arguments.of("r5.2xlarge", 64.0, 8L), - Arguments.of("r5.4xlarge", 128.0, 16L), - Arguments.of("r5.8xlarge", 256.0, 32L), Arguments.of("r5.large", 16.0, 2L), Arguments.of("r5.xlarge", 32.0, 4L), //Different cases with non 0.0 deviation Arguments.of("m5.24xlarge", 384.0, 110L), //"m5.24xlarge", 384.0, 96L has closest coef. - Arguments.of("r5.2xlarge", 150.0, 8L), //"r5.2xlarge", 64.0, 8L has closest coef. - Arguments.of("r5.4xlarge", 128.0, 10L), //"r5.4xlarge", 128.0, 16L has closest coef. Arguments.of("p2.16xlarge", 650.0, 64L), //"p2.16xlarge", 768.0, 64L has closest coef. Arguments.of("p2.8xlarge", 650.0, 32L), //"p2.8xlarge", 488.0, 32L has closest coef. //Check that the default values will be used if the request does not have them, respectively Arguments.of("r5.large", 650.0, null), //defaultCpuCore = 2.0, closest should be "r5.large", 16.0, 2L Arguments.of("c5.xlarge", null, 32L), //defaultRamGb = 8.0, closest should be "c5.xlarge", 8.0, 4L - Arguments.of("m5.large", null, null) //defaultRamGb = 8.0 and defaultCpuCore = 2 => "m5.large", 8.0, 2L + Arguments.of("m5.large", null, null), //defaultRamGb = 8.0 and defaultCpuCore = 2 => "m5.large", 8.0, 2L + + // Check instanceTypes with memoryUnitType (KiB, MiB, GiB, TiB, PiB, EiB) + Arguments.of("r5.2xlarge", 161061273600.0, 8L), //RAM_in_GiB(r5.2xlarge) = 150 * 1073741824.0 = 161061273600.0 + Arguments.of("r5.12xlarge", 384.0, 48L), //RAM_in_GiB(r5.12xlarge) = 384 * 1.0 = 384.0 + Arguments.of("r5.16xlarge", 0.5, 64L), //RAM_in_GiB(r5.16xlarge) = 512 * 0.0009765625 = 0.5 + Arguments.of("r5.24xlarge", 0.00073242187776, 96L), //RAM_in_GiB(r5.24xlarge) = 768 * 0.00000095367432 = 0.00073242187776 + Arguments.of("r5.2xlarge", 68719476736.0, 8L), //RAM_in_GiB(r5.2xlarge) = 64 * 1073741824.0 = 68719476736 + Arguments.of("r5.4xlarge", 134217728.0, 16L), //RAM_in_GiB(r5.2xlarge) = 128 * 1048576.0 = 134217728 + Arguments.of("r5.8xlarge", 262144.0, 32L) //RAM_in_GiB(r5.2xlarge) = 256 * 1024.0 = 262144 ); } private static void fillWithAllowedInstanceTypes(List instanceTypeList) { - instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.12xlarge", 96.0, 48L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.18xlarge", 144.0, 72L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.24xlarge", 192.0, 96L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.2xlarge", 16.0, 8L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.4xlarge", 32.0, 16L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.9xlarge", 72.0, 36L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.large", 4.0, 2L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.metal", 192.0, 96L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.xlarge", 8.0, 4L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.12xlarge", 192.0, 48L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.16xlarge", 256.0, 64L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.24xlarge", 384.0, 96L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.2xlarge", 32.0, 8L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.4xlarge", 64.0, 16L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.8xlarge", 128.0, 32L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.large", 8.0, 2L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.metal", 384.0, 96L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.xlarge", 16.0, 4L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("p2.16xlarge", 768.0, 64L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("p2.8xlarge", 488.0, 32L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("p2.xlarge", 61.0, 4L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.12xlarge", 384.0, 48L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.16xlarge", 512.0, 64L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.24xlarge", 768.0, 96L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.2xlarge", 64.0, 8L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.4xlarge", 128.0, 16L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.8xlarge", 256.0, 32L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.large", 16.0, 2L)); - instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.xlarge", 32.0, 4L)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.12xlarge", 96.0, 48L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.18xlarge", 144.0, 72L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.24xlarge", 192.0, 96L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.2xlarge", 16.0, 8L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.4xlarge", 32.0, 16L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.9xlarge", 72.0, 36L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.large", 4.0, 2L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.metal", 192.0, 96L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("c5.xlarge", 8.0, 4L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.12xlarge", 192.0, 48L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.16xlarge", 256.0, 64L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.24xlarge", 384.0, 96L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.2xlarge", 32.0, 8L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.4xlarge", 64.0, 16L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.8xlarge", 128.0, 32L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.large", 8.0, 2L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.metal", 384.0, 96L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("m5.xlarge", 16.0, 4L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("p2.16xlarge", 768.0, 64L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("p2.8xlarge", 488.0, 32L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("p2.xlarge", 61.0, 4L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.large", 16.0, 2L, GIB)); + + //instances with non-default instanceMemoryUnits + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.xlarge", 32.0, 4L, null)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.24xlarge", 768.0, 96L, "KiB")); + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.16xlarge", 512.0, 64L, "MiB")); + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.12xlarge", 384.0, 48L, GIB)); + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.8xlarge", 256.0, 32L, "TiB")); + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.4xlarge", 128.0, 16L, "PiB")); + instanceTypeList.add(getInstanceWithNameRamAndCpu("r5.2xlarge", 64.0, 8L, "EiB")); } - private static InstanceType getInstanceWithNameRamAndCpu(String instanceName, Double instanceRam, Long instanceCpu) { + private static InstanceType getInstanceWithNameRamAndCpu(String instanceName, Double instanceRam, Long instanceCpu, String memoryType) { InstanceType instanceType = new InstanceType(); instanceType.setName(instanceName); instanceType.setMemory(instanceRam.floatValue()); instanceType.setVCPU(instanceCpu.intValue()); + instanceType.setMemoryUnit(memoryType); return instanceType; } } \ No newline at end of file From 847e7ecacdea54176899159a0bb9e54581c73b56 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 20 Sep 2019 14:06:16 +0300 Subject: [PATCH 123/194] couple clean-code for TaskMapperTest class --- .../epam/pipeline/tesadapter/service/TaskMapperTest.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index 3a3ace3cbd..bf74b8611c 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -187,7 +187,7 @@ public void expectIllegalArgExceptionWhenRunMapToPipelineStartWithNullTool() { @Test() - public void expectIllegalArgExceptionWhenRunGetExecutorFromTesExecutorsList() { + public void expectIllegalArgExceptionWhenRunMapToPipelineStartWithTwoExecutors() { tesExecutors.add(new TesExecutor()); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> taskMapper.mapToPipelineStart(tesTask)); @@ -213,11 +213,6 @@ public void expectIllegalArgExceptionWhenRunGetProperInstanceTypeWithNullMinInst MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, MIN_INSTANCE)); } - @Test - void getProperInstanceTypeMethodShouldUseProperConversionCoefficientForInstanceMemoryUnit() { - - } - @ParameterizedTest @MethodSource("provideInputForGetProperInstanceTypeTest") public void getProperInstanceTypeShouldReturnProperInstanceName(String instanceTypeName, Double ramGb, Long cpuCore) { From 1dc5396aa044a74b2249f9283f7f69259f97febe Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 20 Sep 2019 17:57:44 +0300 Subject: [PATCH 124/194] covered TesTaskServiceImpl class. added corresponded unit-tests, average - 83%, my part - 100% --- .../tesadapter/service/TesTaskService.java | 2 - .../service/TesTaskServiceImpl.java | 9 +- .../service/TesTaskServiceImplTest.java | 124 ++++++++++++++++-- 3 files changed, 117 insertions(+), 18 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java index f621028a9c..83097d88dd 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskService.java @@ -13,8 +13,6 @@ public interface TesTaskService { TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String pageToken, TaskView view); - void stub(); - TesCancelTaskResponse cancelTesTask(String id); TesTask getTesTask(String id, TaskView view); diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 1a72d4041b..8b60863989 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -61,8 +61,8 @@ public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient, TaskMap public TesCreateTaskResponse submitTesTask(TesTask body) { TesCreateTaskResponse tesCreateTaskResponse = new TesCreateTaskResponse(); PipelineRun pipelineRun = cloudPipelineAPIClient.runPipeline(taskMapper.mapToPipelineStart(body)); - Assert.notNull(pipelineRun.getId(), messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_REQUIRED, - ID, TesCreateTaskResponse.class.getSimpleName())); + Assert.notNull(pipelineRun.getId(), messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, + ID)); tesCreateTaskResponse.setId(String.valueOf(pipelineRun.getId())); return tesCreateTaskResponse; } @@ -102,11 +102,6 @@ private List filterRunsWithOutNamePrefix(Long pageSize, String page return cloudPipelineAPIClient.filterRuns(filterVO, LOAD_STORAGE_LINKS).getElements(); } - @Override - public void stub() { - //stubbed method - } - @Override public TesCancelTaskResponse cancelTesTask(String id) { RunStatusVO updateStatus = new RunStatusVO(); diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java index 395a487823..974becd211 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java @@ -1,42 +1,148 @@ package com.epam.pipeline.tesadapter.service; import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.entity.pipeline.TaskStatus; +import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.rest.PagedResult; +import com.epam.pipeline.tesadapter.common.MessageConstants; import com.epam.pipeline.tesadapter.common.MessageHelper; +import com.epam.pipeline.tesadapter.configuration.AppConfiguration; import com.epam.pipeline.tesadapter.entity.TaskView; +import com.epam.pipeline.tesadapter.entity.TesCancelTaskResponse; +import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.vo.PagingRunFilterExpressionVO; -import org.junit.jupiter.api.Assertions; +import com.epam.pipeline.vo.PagingRunFilterVO; +import com.epam.pipeline.vo.RunStatusVO; +import com.epam.pipeline.vo.filter.FilterExpressionTypeVO; +import com.epam.pipeline.vo.filter.FilterExpressionVO; +import com.epam.pipeline.vo.filter.FilterOperandTypeVO; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; import java.util.ArrayList; import java.util.List; +import java.util.stream.Stream; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +@ExtendWith(value = SpringExtension.class) +@ContextConfiguration(classes = {AppConfiguration.class}) +@SuppressWarnings({"unused", "PMD.TooManyStaticImports"}) class TesTaskServiceImplTest { private TesTaskServiceImpl tesTaskService; - private static final String PAGE_TOKEN = "1"; - private static final String NAME_PREFIX = "pipeline"; - private static final Long PAGE_SIZE = 20L; + + @Autowired + private MessageHelper messageHelper; + + private static final String DEFAULT_PAGE_TOKEN = "1"; + private static final String PREFIX_FIELD = "pod.id"; + private static final String PREFIX_NAME = "pipeline-12410"; + private static final Integer NUMBER_OF_RUNS = 20; - private static final TaskView DEFAULT_VIEW = TaskView.BASIC; + private static final TaskView DEFAULT_TASK_VIEW = TaskView.MINIMAL; + private static final Long DEFAULT_PIPELINE_ID = 12410L; + private static final String ID = "id"; + private static final Integer FIRST = 0; + private static final Boolean LOAD_STORAGE_LINKS = true; + private static final Long DEFAULT_PAGE_SIZE = 256L; private List pipelineRunList = new ArrayList<>(); + private TesTask tesTask = new TesTask(); + private PipelineStart pipelineStart = new PipelineStart(); + private PipelineRun pipelineRun = new PipelineRun(); + private PagingRunFilterExpressionVO filterExpressionVO = new PagingRunFilterExpressionVO(); + private FilterExpressionVO expression = new FilterExpressionVO(); + private PagedResult> pagedPipelineRunList = new PagedResult<>(pipelineRunList, NUMBER_OF_RUNS); + private TaskMapper taskMapper = mock(TaskMapper.class); private CloudPipelineAPIClient cloudPipelineAPIClient = mock(CloudPipelineAPIClient.class); @BeforeEach public void setUp() { + when(taskMapper.mapToPipelineStart(tesTask)).thenReturn(pipelineStart); + when(taskMapper.mapToTesTask(pipelineRun, DEFAULT_TASK_VIEW)).thenReturn(tesTask); + when(cloudPipelineAPIClient.runPipeline(pipelineStart)).thenReturn(pipelineRun); when(cloudPipelineAPIClient.searchRuns(any(PagingRunFilterExpressionVO.class))).thenReturn(pagedPipelineRunList); - tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient, mock(TaskMapper.class), - mock(MessageHelper.class)); + when(cloudPipelineAPIClient.filterRuns(any(PagingRunFilterVO.class), eq(LOAD_STORAGE_LINKS))) + .thenReturn(pagedPipelineRunList); + pipelineRun.setId(DEFAULT_PIPELINE_ID); + tesTask.setId(DEFAULT_PIPELINE_ID.toString()); + pipelineRunList.add(pipelineRun); + tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient, taskMapper, messageHelper); + } + + @Test + void listTesTaskShouldReturnNotNullIfRequestDifferent() { + expression.setField(PREFIX_FIELD); + expression.setValue(PREFIX_NAME); + expression.setOperand(FilterOperandTypeVO.EQUALS.getOperand()); + expression.setFilterExpressionType(FilterExpressionTypeVO.LOGICAL); + filterExpressionVO.setFilterExpression(expression); + assertNotNull(tesTaskService.listTesTask(PREFIX_NAME, DEFAULT_PAGE_SIZE, DEFAULT_PAGE_TOKEN, DEFAULT_TASK_VIEW)); + assertNotNull(tesTaskService.listTesTask(null, DEFAULT_PAGE_SIZE, DEFAULT_PAGE_TOKEN, DEFAULT_TASK_VIEW)); + assertNotNull(tesTaskService.listTesTask(null, null, DEFAULT_PAGE_TOKEN, DEFAULT_TASK_VIEW)); + assertNotNull(tesTaskService.listTesTask(null, null, null, DEFAULT_TASK_VIEW)); + assertNotNull(tesTaskService.listTesTask(null, null, null, null)); + + assertEquals(tesTaskService.listTesTask(null, DEFAULT_PAGE_SIZE, DEFAULT_PAGE_TOKEN, DEFAULT_TASK_VIEW) + .getTasks().get(FIRST).getId() + , DEFAULT_PIPELINE_ID.toString()); + } + + @Test + void submitTesTaskShouldReturnTesCreateTaskResponse() { + assertEquals(tesTaskService.submitTesTask(tesTask).getId(), pipelineRun.getId().toString()); + } + + @Test + void expectIllegalArgExceptionWhenRunSubmitTesTaskWithNullIdResponse() { + pipelineRun.setId(null); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> tesTaskService.submitTesTask(tesTask)); + assertTrue(exception.getMessage().contains(messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, ID))); } @Test - void listTesTask() { - Assertions.assertNotNull(tesTaskService.listTesTask(NAME_PREFIX, PAGE_SIZE, PAGE_TOKEN, DEFAULT_VIEW)); + void cancelTesTaskShouldStopTaskWithCorrespondId() { + RunStatusVO updateStatus = new RunStatusVO(); + updateStatus.setStatus(TaskStatus.STOPPED); + pipelineRun.setStatus(TaskStatus.STOPPED); + when(cloudPipelineAPIClient.updateRunStatus(DEFAULT_PIPELINE_ID, updateStatus)).thenReturn(pipelineRun); + assertEquals(new TesCancelTaskResponse(), tesTaskService.cancelTesTask(DEFAULT_PIPELINE_ID.toString())); + } + + @ParameterizedTest + @MethodSource("provideWrongIdInputForCancelTesTask") + void expectIllegalStateExceptionWhenRunCancelTesTaskWithWrongId(String id) { + IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> tesTaskService.cancelTesTask(id)); + assertEquals(exception.getMessage(), messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, ID)); + } + + private static Stream provideWrongIdInputForCancelTesTask() { + return Stream.of( + Arguments.of("justText"), + Arguments.of("12,44"), + Arguments.of("1456_44"), + Arguments.of("taskId-12410"), + Arguments.of("14567 "), + Arguments.of("null") + ); } } \ No newline at end of file From 5176d0aa85c92b296e0da7d7c5c4ae9d4b6ca16e Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 20 Sep 2019 18:24:03 +0300 Subject: [PATCH 125/194] removed unnecessary unit-test from TesTaskServiceImplTest class, also added for searchRuns()/filterRuns() non-null response if filtering without any success --- .../service/TesTaskServiceImpl.java | 4 ++-- .../service/TesTaskServiceImplTest.java | 21 ------------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 8b60863989..41b58de393 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -92,14 +92,14 @@ private List searchRunsWithNamePrefix(String namePrefix, Long pageS filterExpressionVO.setFilterExpression(expression); filterExpressionVO.setPage(Integer.parseInt(Optional.ofNullable(pageToken).orElse(DEFAULT_PAGE_TOKEN))); filterExpressionVO.setPageSize(Optional.ofNullable(pageSize).orElse(DEFAULT_PAGE_SIZE).intValue()); - return cloudPipelineAPIClient.searchRuns(filterExpressionVO).getElements(); + return ListUtils.emptyIfNull(cloudPipelineAPIClient.searchRuns(filterExpressionVO).getElements()); } private List filterRunsWithOutNamePrefix(Long pageSize, String pageToken) { PagingRunFilterVO filterVO = new PagingRunFilterVO(); filterVO.setPage(Integer.parseInt(Optional.ofNullable(pageToken).orElse(DEFAULT_PAGE_TOKEN))); filterVO.setPageSize(Optional.ofNullable(pageSize).orElse(DEFAULT_PAGE_SIZE).intValue()); - return cloudPipelineAPIClient.filterRuns(filterVO, LOAD_STORAGE_LINKS).getElements(); + return ListUtils.emptyIfNull(cloudPipelineAPIClient.filterRuns(filterVO, LOAD_STORAGE_LINKS).getElements()); } @Override diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java index 974becd211..7ad772d99b 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java @@ -13,9 +13,7 @@ import com.epam.pipeline.vo.PagingRunFilterExpressionVO; import com.epam.pipeline.vo.PagingRunFilterVO; import com.epam.pipeline.vo.RunStatusVO; -import com.epam.pipeline.vo.filter.FilterExpressionTypeVO; import com.epam.pipeline.vo.filter.FilterExpressionVO; -import com.epam.pipeline.vo.filter.FilterOperandTypeVO; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -31,7 +29,6 @@ import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -85,24 +82,6 @@ public void setUp() { tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient, taskMapper, messageHelper); } - @Test - void listTesTaskShouldReturnNotNullIfRequestDifferent() { - expression.setField(PREFIX_FIELD); - expression.setValue(PREFIX_NAME); - expression.setOperand(FilterOperandTypeVO.EQUALS.getOperand()); - expression.setFilterExpressionType(FilterExpressionTypeVO.LOGICAL); - filterExpressionVO.setFilterExpression(expression); - assertNotNull(tesTaskService.listTesTask(PREFIX_NAME, DEFAULT_PAGE_SIZE, DEFAULT_PAGE_TOKEN, DEFAULT_TASK_VIEW)); - assertNotNull(tesTaskService.listTesTask(null, DEFAULT_PAGE_SIZE, DEFAULT_PAGE_TOKEN, DEFAULT_TASK_VIEW)); - assertNotNull(tesTaskService.listTesTask(null, null, DEFAULT_PAGE_TOKEN, DEFAULT_TASK_VIEW)); - assertNotNull(tesTaskService.listTesTask(null, null, null, DEFAULT_TASK_VIEW)); - assertNotNull(tesTaskService.listTesTask(null, null, null, null)); - - assertEquals(tesTaskService.listTesTask(null, DEFAULT_PAGE_SIZE, DEFAULT_PAGE_TOKEN, DEFAULT_TASK_VIEW) - .getTasks().get(FIRST).getId() - , DEFAULT_PIPELINE_ID.toString()); - } - @Test void submitTesTaskShouldReturnTesCreateTaskResponse() { assertEquals(tesTaskService.submitTesTask(tesTask).getId(), pipelineRun.getId().toString()); From d5f3bf119c373fd3b1ea1610322373b82b03168d Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Mon, 23 Sep 2019 11:19:47 +0300 Subject: [PATCH 126/194] added explicit scoping for tests --- .../tesadapter/service/TesTaskServiceImplTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java index 7ad772d99b..7f261c5e63 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java @@ -83,12 +83,12 @@ public void setUp() { } @Test - void submitTesTaskShouldReturnTesCreateTaskResponse() { + public void submitTesTaskShouldReturnTesCreateTaskResponse() { assertEquals(tesTaskService.submitTesTask(tesTask).getId(), pipelineRun.getId().toString()); } @Test - void expectIllegalArgExceptionWhenRunSubmitTesTaskWithNullIdResponse() { + public void expectIllegalArgExceptionWhenRunSubmitTesTaskWithNullIdResponse() { pipelineRun.setId(null); IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> tesTaskService.submitTesTask(tesTask)); @@ -97,7 +97,7 @@ void expectIllegalArgExceptionWhenRunSubmitTesTaskWithNullIdResponse() { } @Test - void cancelTesTaskShouldStopTaskWithCorrespondId() { + public void cancelTesTaskShouldStopTaskWithCorrespondId() { RunStatusVO updateStatus = new RunStatusVO(); updateStatus.setStatus(TaskStatus.STOPPED); pipelineRun.setStatus(TaskStatus.STOPPED); @@ -107,7 +107,7 @@ void cancelTesTaskShouldStopTaskWithCorrespondId() { @ParameterizedTest @MethodSource("provideWrongIdInputForCancelTesTask") - void expectIllegalStateExceptionWhenRunCancelTesTaskWithWrongId(String id) { + public void expectIllegalStateExceptionWhenRunCancelTesTaskWithWrongId(String id) { IllegalStateException exception = assertThrows(IllegalStateException.class, () -> tesTaskService.cancelTesTask(id)); assertEquals(exception.getMessage(), messageHelper.getMessage( From 7c29e6eb9aeb2a5126070c791cf148e2a84ef16a Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Mon, 23 Sep 2019 15:35:07 +0300 Subject: [PATCH 127/194] added not ready testCreateTesResources --- .../pipeline/entity/cluster/InstanceType.java | 2 + .../tesadapter/entity/TesResources.java | 2 + .../tesadapter/service/TaskMapper.java | 4 - .../tesadapter/service/TaskMapperTest.java | 75 ++++++++++++++----- 4 files changed, 59 insertions(+), 24 deletions(-) diff --git a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/entity/cluster/InstanceType.java b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/entity/cluster/InstanceType.java index 499e178009..121b303188 100644 --- a/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/entity/cluster/InstanceType.java +++ b/cloud-pipeline-common/model/src/main/java/com/epam/pipeline/entity/cluster/InstanceType.java @@ -18,6 +18,7 @@ import lombok.AllArgsConstructor; import lombok.Builder; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -27,6 +28,7 @@ @Builder @AllArgsConstructor @NoArgsConstructor +@EqualsAndHashCode public class InstanceType { private String sku; private String name; diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java index 812da16e0c..328518ecf4 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesResources.java @@ -6,6 +6,7 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import javax.validation.Valid; @@ -15,6 +16,7 @@ @Builder @NoArgsConstructor @AllArgsConstructor +@EqualsAndHashCode @ApiModel(description = "Resources describes the resources requested by a task.") public class TesResources { @ApiModelProperty(value = "Requested number of CPUs") diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 8169a2571a..5ba36df7f0 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -215,10 +215,6 @@ Double convertMemoryUnitTypeToGiB(String memoryUnit) { } public TesTask mapToTesTask(PipelineRun run, TaskView view) { - return filterTesTaskWithView(run, view); - } - - private TesTask filterTesTaskWithView(PipelineRun run, TaskView view) { final TesTask.TesTaskBuilder tesTask = TesTask.builder() .id(String.valueOf(run.getId())) .state(createTesState(run)); diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index 8d8207ab9d..17816bd22e 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -26,12 +26,12 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -43,7 +43,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; @@ -396,28 +395,64 @@ private InstanceType getInstanceType(){ } @Test - public void testGetInstanceType(){ - AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = - new AllowedInstanceAndPriceTypes(Collections.singletonList(getInstanceType()), - Collections.singletonList(getInstanceType()), Collections.singletonList("string")); + public void testGetInstanceType() { when(cloudPipelineAPIClient.loadTool(anyString())).thenReturn(mock(Tool.class)); - when(cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(anyLong(), anyLong(), anyBoolean())).thenReturn(allowedInstanceAndPriceTypes); + when(cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(anyLong(), anyLong(), + anyBoolean())).thenReturn(getAllowedInstanceAndPriceTypes()); InstanceType instanceType = getInstanceType(); - assertEquals(instanceType.getName(), taskMapper.getInstanceType(getPipelineRun()).getName()); - assertEquals(instanceType.getMemory(), taskMapper.getInstanceType(getPipelineRun()).getMemory()); - assertEquals(instanceType.getMemoryUnit(), taskMapper.getInstanceType(getPipelineRun()).getMemoryUnit()); - assertEquals(instanceType.getVCPU(), taskMapper.getInstanceType(getPipelineRun()).getVCPU()); + assertEquals(instanceType, taskMapper.getInstanceType(getPipelineRun())); + } + + @Test + public void testCreateTesResources() { + TesResources tesResources = TesResources.builder() + .preemptible(getPipelineRun().getInstance().getSpot()) + .diskGb(new Double(getPipelineRun().getInstance().getNodeDisk())) + .ramGb(getInstanceType().getMemory() * taskMapper.convertMemoryUnitTypeToGiB(getInstanceType().getMemoryUnit())) + .cpuCores((long) getInstanceType().getVCPU()) + .zones(Collections.singletonList(DEFAULT_REGION_NAME)) + .build(); + when(cloudPipelineAPIClient.loadRegion(Mockito.eq(getPipelineRun().getInstance().getCloudRegionId())).getRegionCode()).thenReturn(DEFAULT_REGION_NAME); + + when(cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(anyLong(), anyLong(), + getPipelineRun().getInstance().getSpot())).thenReturn(getAllowedInstanceAndPriceTypes()); + assertEquals(tesResources, taskMapper.createTesResources(getPipelineRun())); + } + + private TesResources getTesResources() { + return TesResources.builder() + .preemptible(getPipelineRun().getInstance().getSpot()) + .diskGb(new Double(getPipelineRun().getInstance().getNodeDisk())) + .ramGb(getInstanceType().getMemory() * taskMapper.convertMemoryUnitTypeToGiB(getInstanceType().getMemoryUnit())) + .cpuCores((long) getInstanceType().getVCPU()) + .zones(Collections.singletonList(DEFAULT_REGION_NAME)) + .build(); + } + + private AllowedInstanceAndPriceTypes getAllowedInstanceAndPriceTypes() { + return new AllowedInstanceAndPriceTypes(Collections.singletonList(getInstanceType()), + Collections.singletonList(getInstanceType()), Collections.singletonList("string")); } // @Test -// public void testCreateTesResources(){ -// TesResources tesResources = TesResources.builder() -// .preemptible(getPipelineRun().getInstance().getSpot()) -// .diskGb(new Double(getPipelineRun().getInstance().getNodeDisk())) -// .ramGb(getInstanceType().getMemory() * taskMapper.convertMemoryUnitTypeToGiB(getInstanceType().getMemoryUnit())) -// .cpuCores((long) getInstanceType().getVCPU()) -// .zones(Collections.singletonList(DEFAULT_REGION_NAME)) -// .build(); -// assertEquals(tesResources, taskMapper.createTesResources(getPipelineRun())); +// public void getMapToTesTask(){ +// assertEquals(TesTask.builder() +// .id(String.valueOf(getPipelineRun().getId())) +// .state(TesState.RUNNING).build(), taskMapper.mapToTesTask(getPipelineRun(), TaskView.MINIMAL)); +// +// when(cloudPipelineAPIClient.loadPipelineTasks(getPipelineRun().getId())).thenReturn(getPipelineTaskListWithConsole()); +// when(cloudPipelineAPIClient.loadTool(anyString())).thenReturn(new Tool()); +// List tesExecutors = new ArrayList<>(); +// tesExecutors.add(new TesExecutor()); +// tesExecutors.add(new TesExecutor()); +// assertEquals(TesTask.builder() +// .id(String.valueOf(getPipelineRun().getId())) +// .state(TesState.RUNNING) +// .name(getPipelineRun().getPodId()) +// .resources(getTesResources()) +// .executors(tesExecutors) +// .outputs(Collections.singletonList(new TesOutput())) +// .creationTime(getPipelineRun().getStartDate().toString()) +// .build(), taskMapper.mapToTesTask(getPipelineRun(), TaskView.BASIC)); // } } \ No newline at end of file From 7656579f07903bf095143ddbf16124525636e86b Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 24 Sep 2019 15:40:25 +0300 Subject: [PATCH 128/194] added test mapToTes and test for TesState --- .../tesadapter/service/TaskMapperTest.java | 344 +++++++++--------- 1 file changed, 173 insertions(+), 171 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index 17816bd22e..e6d08c5b01 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -10,11 +10,16 @@ import com.epam.pipeline.entity.pipeline.Tool; import com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter; import com.epam.pipeline.entity.region.AbstractCloudRegion; +import com.epam.pipeline.entity.region.AwsRegion; import com.epam.pipeline.tesadapter.common.MessageConstants; import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.configuration.AppConfiguration; +import com.epam.pipeline.tesadapter.entity.PipelineDiskMemoryTypes; +import com.epam.pipeline.tesadapter.entity.TaskView; import com.epam.pipeline.tesadapter.entity.TesExecutor; import com.epam.pipeline.tesadapter.entity.TesExecutorLog; +import com.epam.pipeline.tesadapter.entity.TesInput; +import com.epam.pipeline.tesadapter.entity.TesOutput; import com.epam.pipeline.tesadapter.entity.TesResources; import com.epam.pipeline.tesadapter.entity.TesState; import com.epam.pipeline.tesadapter.entity.TesTask; @@ -26,7 +31,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -36,13 +40,12 @@ import java.util.Date; import java.util.HashMap; import java.util.List; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; @@ -61,27 +64,51 @@ class TaskMapperTest { private static final String EXECUTORS = "executors"; private static final Long STUBBED_REGION_ID = 1L; private static final Long STUBBED_TOOL_ID = 11584L; + private static final Long TOOL_ID = 1l; private static final String STUBBED_IMAGE = "cp-docker-registry.default.svc.cluster.local:31443/library/centos:latest"; + private static final String INPUT = "input"; + private static final String OUTPUT = "output"; + private static final Double KIB_TO_GIB = 0.00000095367432; + private static final String STUBBED_NAME_PIPELINE_PARAMETER = "pipelineRunParameter"; + private static final String STUBBED_VALUE_PIPELINE_PARAMETER = "www.url.ru"; + private static final String OUTPUT_LOG_STRING_FORMAT = "%s - %s - %s"; + private static final String TYPE_NODE = "nodeType"; + private static final String STUBBED_TEXT_FOR_LOG = "log"; + private static final String PIPELINE_TASK_NAME_CONSOLE = "Console"; + private static final String PIPELINE_TASK_NAME_InitializeEnvironment = "InitializeEnvironment"; + private static final String ANY_STRING = "any_string"; private static List allowedInstanceTypes; @Autowired private MessageHelper messageHelper; - private TaskMapper taskMapper; + private static TaskMapper taskMapper; + private static PipelineRun run = getPipelineRun(); private List zones = new ArrayList<>(); private List abstractCloudRegions = new ArrayList<>(); private AbstractCloudRegion abstractCloudRegion = mock(AbstractCloudRegion.class); - private CloudPipelineAPIClient cloudPipelineAPIClient = mock(CloudPipelineAPIClient.class); + private static CloudPipelineAPIClient cloudPipelineAPIClient = mock(CloudPipelineAPIClient.class); private AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = mock(AllowedInstanceAndPriceTypes.class); private Tool tool = mock(Tool.class); private TesTask tesTask = mock(TesTask.class); private PipelineRun pipelineRun = mock(PipelineRun.class); + @BeforeAll public static void setUpAll() { allowedInstanceTypes = new ArrayList<>(); fillWithAllowedInstanceTypes(allowedInstanceTypes); + + AwsRegion awsRegion = new AwsRegion(); + awsRegion.setRegionCode(DEFAULT_REGION_NAME); + when(cloudPipelineAPIClient.loadRegion(anyLong())).thenReturn(awsRegion); + when(cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(TOOL_ID, 1l, + true)).thenReturn(getAllowedInstanceAndPriceTypes(run)); + Tool tool = new Tool(); + tool.setId(TOOL_ID); + tool.setName(STUBBED_IMAGE); + when(cloudPipelineAPIClient.loadTool(anyString())).thenReturn(tool); } @BeforeEach @@ -241,83 +268,90 @@ private static InstanceType getInstanceWithNameRamAndCpu(String instanceName, Do return instanceType; } - @Test - public void testCreateTesState() { - when(pipelineRun.getStatus()).thenReturn(TaskStatus.RUNNING); - assertEquals(TesState.RUNNING, taskMapper.createTesState(pipelineRun)); - - when(pipelineRun.getStatus()).thenReturn(TaskStatus.STOPPED); - assertEquals(TesState.CANCELED, taskMapper.createTesState(pipelineRun)); - - when(pipelineRun.getStatus()).thenReturn(TaskStatus.PAUSED); - assertEquals(TesState.PAUSED, taskMapper.createTesState(pipelineRun)); - - List pipelineTaskListWithConsole = getPipelineTaskListWithConsole(); - when(cloudPipelineAPIClient.loadPipelineTasks(anyLong())).thenReturn(pipelineTaskListWithConsole); - assertEquals(TesState.QUEUED, taskMapper.createTesState(getPipelineRun())); - - List pipelineTaskListWithEnv = getPipelineTaskListWithEnv(); - when(cloudPipelineAPIClient.loadPipelineTasks(anyLong())).thenReturn(pipelineTaskListWithEnv); - assertEquals(TesState.RUNNING, taskMapper.createTesState(getPipelineRun())); - - List pipelineTaskListWithoutEnv = getPipelineTaskListWithoutEnv(); - when(cloudPipelineAPIClient.loadPipelineTasks(anyLong())).thenReturn(pipelineTaskListWithoutEnv); - assertEquals(TesState.INITIALIZING, taskMapper.createTesState(getPipelineRun())); - - when(pipelineRun.getStatus()).thenReturn(TaskStatus.RESUMING); - assertEquals(TesState.UNKNOWN, taskMapper.createTesState(pipelineRun)); - - when(pipelineRun.getStatus()).thenReturn(TaskStatus.SUCCESS); - assertEquals(TesState.COMPLETE, taskMapper.createTesState(pipelineRun)); - - when(pipelineRun.getStatus()).thenReturn(TaskStatus.FAILURE); - assertEquals(TesState.EXECUTOR_ERROR, taskMapper.createTesState(pipelineRun)); - - assertNotNull(taskMapper.createTesState(pipelineRun)); - } - - private List getPipelineTaskListWithConsole(){ - ArrayList pipelineTasksList = new ArrayList<>(); - PipelineTask pipelineTaskConsole = new PipelineTask(); - pipelineTaskConsole.setName("Console"); - pipelineTasksList.add(pipelineTaskConsole); - return pipelineTasksList; - } - - private List getPipelineTaskListWithEnv(){ - ArrayList pipelineTasksList = new ArrayList<>(); - PipelineTask pipelineTaskInitializeEnvironment = new PipelineTask(); - pipelineTaskInitializeEnvironment.setName("InitializeEnvironment"); - pipelineTasksList.add(pipelineTaskInitializeEnvironment); - return pipelineTasksList; + @ParameterizedTest + @MethodSource("provideTesTaskForTestTaskMapper") + public void testTaskMapperMapPipelineRunToTesTask(PipelineRun run, TaskView view, TesTask tesTaskExpected) { + TesTask tesTaskActual = taskMapper.mapToTesTask(run, view); + assertEquals(tesTaskExpected, tesTaskActual); } - private List getPipelineTaskListWithoutEnv(){ - List pipelineTasksList = getPipelineTaskListWithConsole(); - PipelineTask pipelineTask = new PipelineTask(); - pipelineTask.setName("someState"); - pipelineTasksList.add(pipelineTask); - return pipelineTasksList; + private static Stream provideTesTaskForTestTaskMapper() { + //data for TaskView.Minimal + TesTask tesTaskForMinimal = TesTask.builder() + .id(String.valueOf(run.getId())) + .state(TesState.RUNNING) + .build(); + //data for TaskView.basic + TesTask tesTaskForBasic = TesTask.builder() + .id(String.valueOf(run.getId())) + .state(TesState.RUNNING) + .resources(createResources(run)) + .executors(createListExecutor(run)) + .outputs(getListPipelineRunParameter().stream() + .filter(pipelineRunParameter -> pipelineRunParameter.getType().contains(OUTPUT)) + .map(pipelineRunParameter -> + TesOutput.builder() + .name(pipelineRunParameter.getName()) + .url(pipelineRunParameter.getValue()) + .build()).collect(Collectors.toList())) + .creationTime(run.getStartDate().toString()) + .build(); + //data for TaskView.FULL + TesTask tesTaskForFull = TesTask.builder() + .id(String.valueOf(run.getId())) + .state(TesState.RUNNING) + .resources(createResources(run)) + .executors(createListExecutor(run)) + .outputs(getListPipelineRunParameter().stream() + .filter(pipelineRunParameter -> pipelineRunParameter.getType().contains(OUTPUT)) + .map(pipelineRunParameter -> + TesOutput.builder() + .name(pipelineRunParameter.getName()) + .url(pipelineRunParameter.getValue()) + .build()).collect(Collectors.toList())) + .creationTime(run.getStartDate().toString()) + .inputs(getListPipelineRunParameter().stream() + .filter(pipelineRunParameter -> pipelineRunParameter.getType().contains(INPUT)) + .map(pipelineRunParameter -> + TesInput.builder() + .name(pipelineRunParameter.getName()) + .url(pipelineRunParameter.getValue()) + .build()).collect(Collectors.toList())) + .logs(createTesTaskLog(run.getId())) + .build(); + return Stream.of( + Arguments.of(run, TaskView.MINIMAL, tesTaskForMinimal), + Arguments.of(run, TaskView.BASIC, tesTaskForBasic), + Arguments.of(run, TaskView.FULL, tesTaskForFull) + ); } - @Test - public void testCreateTesInput() { - assertNotNull(taskMapper.createTesInput(pipelineRun.getPipelineRunParameters())); - assertEquals("pipelineRunParameter", taskMapper.createTesInput(getListPipelineRunParameter()).get(0).getName()); - assertEquals("www.url.ru", taskMapper.createTesInput(getListPipelineRunParameter()).get(0).getUrl()); - + private static PipelineRun getPipelineRun() { + PipelineRun pipelineRun = new PipelineRun(); + pipelineRun.setId(1L); + pipelineRun.setStatus(TaskStatus.RUNNING); + pipelineRun.setPodIP("pipelineRun"); + pipelineRun.setPipelineRunParameters(getListPipelineRunParameter()); + pipelineRun.setStartDate(new Date(2000)); + pipelineRun.setInstance(getRunInstance()); + pipelineRun.setActualCmd(ANY_STRING); + HashMap env = new HashMap<>(); + env.put(ANY_STRING, ANY_STRING); + pipelineRun.setEnvVars(env); + pipelineRun.setDockerImage("centos"); + return pipelineRun; } - private List getListPipelineRunParameter() { + private static List getListPipelineRunParameter() { PipelineRunParameter pipelineRunParameterInput = new PipelineRunParameter(); - pipelineRunParameterInput.setType("input"); - pipelineRunParameterInput.setName("pipelineRunParameter"); - pipelineRunParameterInput.setValue("www.url.ru"); + pipelineRunParameterInput.setType(INPUT); + pipelineRunParameterInput.setName(STUBBED_NAME_PIPELINE_PARAMETER); + pipelineRunParameterInput.setValue(STUBBED_VALUE_PIPELINE_PARAMETER); PipelineRunParameter pipelineRunParameterOuput = new PipelineRunParameter(); - pipelineRunParameterOuput.setType("output"); - pipelineRunParameterOuput.setName("pipelineRunParameter"); - pipelineRunParameterOuput.setValue("www.url.ru"); + pipelineRunParameterOuput.setType(OUTPUT); + pipelineRunParameterOuput.setName(STUBBED_NAME_PIPELINE_PARAMETER); + pipelineRunParameterOuput.setValue(STUBBED_VALUE_PIPELINE_PARAMETER); List pipelineRunParameterList = new ArrayList<>(); pipelineRunParameterList.add(pipelineRunParameterInput); @@ -325,134 +359,102 @@ private List getListPipelineRunParameter() { return pipelineRunParameterList; } - @Test - public void testCreateTesOutput() { - assertNotNull(taskMapper.createTesOutput(pipelineRun.getPipelineRunParameters())); - assertEquals("pipelineRunParameter", taskMapper.createTesOutput(getListPipelineRunParameter()).get(0).getName()); - assertEquals("www.url.ru", taskMapper.createTesOutput(getListPipelineRunParameter()).get(0).getUrl()); + private static TesResources createResources(PipelineRun run) { + InstanceType instanceType = getInstanceType(run); + RunInstance runInstance = getRunInstance(); + return TesResources.builder() + .preemptible(runInstance.getSpot()) + .diskGb(new Double(runInstance.getNodeDisk())) + .ramGb(instanceType.getMemory() * KIB_TO_GIB) + .cpuCores((long) instanceType.getVCPU()) + .zones(Collections.singletonList(DEFAULT_REGION_NAME)) + .build(); } - @Test - public void testCreateListExecutor() { - List listTesExecutor = new ArrayList<>(); - listTesExecutor.add(TesExecutor.builder() - .command(Collections.singletonList(getPipelineRun().getActualCmd())) - .env(getPipelineRun().getEnvVars()) - .image(getPipelineRun().getDockerImage()) + private static List createListExecutor(PipelineRun run) { + return Collections.singletonList(TesExecutor.builder() + .command(Collections.singletonList(run.getActualCmd())) + .env(run.getEnvVars()) + .image(run.getDockerImage()) .build()); - assertEquals(listTesExecutor, taskMapper.createListExecutor(getPipelineRun())); } - @Test - public void testCreateTesTaskLog(){ + private static List createTesTaskLog(final Long runId) { RunLog runLog = RunLog.builder() - .date(new Date(12,12,12)) - .taskName("log") - .logText("log") + .date(new Date(12, 12, 12)) + .taskName(STUBBED_TEXT_FOR_LOG) + .logText(STUBBED_TEXT_FOR_LOG) .build(); when(cloudPipelineAPIClient.getRunLog(anyLong())).thenReturn(Collections.singletonList(runLog)); List tesExecutorLogList = Collections.singletonList(TesExecutorLog.builder() - .stdout(String.format("%s - %s - %s", runLog.getDate(), runLog.getTaskName(), runLog.getLogText())) + .stdout(String.format(OUTPUT_LOG_STRING_FORMAT, runLog.getDate(), runLog.getTaskName(), runLog.getLogText())) .build()); List tesTaskLogsList = Collections.singletonList(TesTaskLog.builder() .logs(tesExecutorLogList) .build()); - assertEquals(tesTaskLogsList, taskMapper.createTesTaskLog(anyLong())); + return tesTaskLogsList; } - private PipelineRun getPipelineRun() { - PipelineRun pipelineRun = new PipelineRun(); - pipelineRun.setId(1L); - pipelineRun.setStatus(TaskStatus.RUNNING); - pipelineRun.setPodIP("pipelineRun"); - pipelineRun.setPipelineRunParameters(getListPipelineRunParameter()); - pipelineRun.setStartDate(new Date(2000)); - pipelineRun.setInstance(getRunInstance()); - pipelineRun.setActualCmd("cmd"); - HashMap env = new HashMap<>(); - env.put("env", "env"); - pipelineRun.setEnvVars(env); - pipelineRun.setDockerImage("centos"); - return pipelineRun; - } - - private RunInstance getRunInstance() { + private static RunInstance getRunInstance() { RunInstance runInstance = new RunInstance(); - runInstance.setNodeType("nodeType"); - runInstance.setSpot(true); - runInstance.setNodeDisk(1000); - runInstance.setCloudRegionId(1L); + runInstance.setNodeType(TYPE_NODE); + runInstance.setSpot(DEFAULT_PREEMPTIBLE); + runInstance.setNodeDisk(DEFAULT_HDD_SIZE); + runInstance.setCloudRegionId(STUBBED_REGION_ID); return runInstance; } - private InstanceType getInstanceType(){ + private static InstanceType getInstanceType(PipelineRun run) { return InstanceType.builder() - .name(getPipelineRun().getInstance().getNodeType()) - .memory(10) - .memoryUnit("KiB") - .vCPU(8) + .name(run.getInstance().getNodeType()) + .memory(DEFAULT_RAM_GB.floatValue()) + .memoryUnit(PipelineDiskMemoryTypes.KIB.getValue()) + .vCPU(DEFAULT_CPU_CORES.intValue()) .build(); } - @Test - public void testGetInstanceType() { - when(cloudPipelineAPIClient.loadTool(anyString())).thenReturn(mock(Tool.class)); - when(cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(anyLong(), anyLong(), - anyBoolean())).thenReturn(getAllowedInstanceAndPriceTypes()); - InstanceType instanceType = getInstanceType(); - assertEquals(instanceType, taskMapper.getInstanceType(getPipelineRun())); + private static AllowedInstanceAndPriceTypes getAllowedInstanceAndPriceTypes(PipelineRun run) { + InstanceType instanceType = getInstanceType(run); + return new AllowedInstanceAndPriceTypes(Collections.singletonList(instanceType), + Collections.singletonList(instanceType), Collections.singletonList(ANY_STRING)); } @Test - public void testCreateTesResources() { - TesResources tesResources = TesResources.builder() - .preemptible(getPipelineRun().getInstance().getSpot()) - .diskGb(new Double(getPipelineRun().getInstance().getNodeDisk())) - .ramGb(getInstanceType().getMemory() * taskMapper.convertMemoryUnitTypeToGiB(getInstanceType().getMemoryUnit())) - .cpuCores((long) getInstanceType().getVCPU()) - .zones(Collections.singletonList(DEFAULT_REGION_NAME)) - .build(); - when(cloudPipelineAPIClient.loadRegion(Mockito.eq(getPipelineRun().getInstance().getCloudRegionId())).getRegionCode()).thenReturn(DEFAULT_REGION_NAME); + public void testTesStateForPipelineTaskWithOneTask() { + ArrayList listPipelineTasks = new ArrayList<>(); + PipelineTask pipelineTask = new PipelineTask(); + pipelineTask.setName(PIPELINE_TASK_NAME_CONSOLE); + listPipelineTasks.add(pipelineTask); - when(cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(anyLong(), anyLong(), - getPipelineRun().getInstance().getSpot())).thenReturn(getAllowedInstanceAndPriceTypes()); - assertEquals(tesResources, taskMapper.createTesResources(getPipelineRun())); + when(cloudPipelineAPIClient.loadPipelineTasks(run.getId())).thenReturn(listPipelineTasks); + assertEquals(TesState.QUEUED, taskMapper.mapToTesTask(run, TaskView.MINIMAL).getState()); } - private TesResources getTesResources() { - return TesResources.builder() - .preemptible(getPipelineRun().getInstance().getSpot()) - .diskGb(new Double(getPipelineRun().getInstance().getNodeDisk())) - .ramGb(getInstanceType().getMemory() * taskMapper.convertMemoryUnitTypeToGiB(getInstanceType().getMemoryUnit())) - .cpuCores((long) getInstanceType().getVCPU()) - .zones(Collections.singletonList(DEFAULT_REGION_NAME)) - .build(); + @Test + public void testTesStateForPipelineTaskWithTwoTasksButWithoutInitializeEnvironment() { + ArrayList listPipelineTasks = new ArrayList<>(); + PipelineTask pipelineTaskConsole = new PipelineTask(); + pipelineTaskConsole.setName(PIPELINE_TASK_NAME_CONSOLE); + PipelineTask pipelineTaskAny = new PipelineTask(); + pipelineTaskAny.setName(ANY_STRING); + listPipelineTasks.add(pipelineTaskConsole); + listPipelineTasks.add(pipelineTaskAny); + + when(cloudPipelineAPIClient.loadPipelineTasks(run.getId())).thenReturn(listPipelineTasks); + assertEquals(TesState.INITIALIZING, taskMapper.mapToTesTask(run, TaskView.MINIMAL).getState()); } - private AllowedInstanceAndPriceTypes getAllowedInstanceAndPriceTypes() { - return new AllowedInstanceAndPriceTypes(Collections.singletonList(getInstanceType()), - Collections.singletonList(getInstanceType()), Collections.singletonList("string")); - } + @Test + public void testTesStateForPipelineTaskWithTwoTasksWithInitializeEnvironment() { + ArrayList listPipelineTasks = new ArrayList<>(); + PipelineTask pipelineTaskConsole = new PipelineTask(); + pipelineTaskConsole.setName(PIPELINE_TASK_NAME_CONSOLE); + PipelineTask pipelineTaskInitializeEnvironment = new PipelineTask(); + pipelineTaskInitializeEnvironment.setName(PIPELINE_TASK_NAME_InitializeEnvironment); + listPipelineTasks.add(pipelineTaskConsole); + listPipelineTasks.add(pipelineTaskInitializeEnvironment); -// @Test -// public void getMapToTesTask(){ -// assertEquals(TesTask.builder() -// .id(String.valueOf(getPipelineRun().getId())) -// .state(TesState.RUNNING).build(), taskMapper.mapToTesTask(getPipelineRun(), TaskView.MINIMAL)); -// -// when(cloudPipelineAPIClient.loadPipelineTasks(getPipelineRun().getId())).thenReturn(getPipelineTaskListWithConsole()); -// when(cloudPipelineAPIClient.loadTool(anyString())).thenReturn(new Tool()); -// List tesExecutors = new ArrayList<>(); -// tesExecutors.add(new TesExecutor()); -// tesExecutors.add(new TesExecutor()); -// assertEquals(TesTask.builder() -// .id(String.valueOf(getPipelineRun().getId())) -// .state(TesState.RUNNING) -// .name(getPipelineRun().getPodId()) -// .resources(getTesResources()) -// .executors(tesExecutors) -// .outputs(Collections.singletonList(new TesOutput())) -// .creationTime(getPipelineRun().getStartDate().toString()) -// .build(), taskMapper.mapToTesTask(getPipelineRun(), TaskView.BASIC)); -// } + when(cloudPipelineAPIClient.loadPipelineTasks(run.getId())).thenReturn(listPipelineTasks); + assertEquals(TesState.RUNNING, taskMapper.mapToTesTask(run, TaskView.MINIMAL).getState()); + } } \ No newline at end of file From a26efbdca52ba0c2fc8fa6c76c5fba9d715dbba0 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 24 Sep 2019 15:52:12 +0300 Subject: [PATCH 129/194] added "private" for all methods --- .../pipeline/tesadapter/service/TaskMapper.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 5ba36df7f0..bed2532fa0 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -238,7 +238,7 @@ public TesTask mapToTesTask(PipelineRun run, TaskView view) { } } - TesState createTesState(PipelineRun run) { + private TesState createTesState(PipelineRun run) { List pipelineTaskList = cloudPipelineAPIClient.loadPipelineTasks(run.getId()); switch (run.getStatus()) { case RUNNING: @@ -264,7 +264,7 @@ TesState createTesState(PipelineRun run) { } } - List createTesInput(List parameters) { + private List createTesInput(List parameters) { return parameters.stream() .filter(pipelineRunParameter -> pipelineRunParameter.getType().contains(INPUT_TYPE)) .map(pipelineRunParameter -> @@ -274,7 +274,7 @@ List createTesInput(List parameters) { .build()).collect(Collectors.toList()); } - List createTesOutput(List parameters) { + private List createTesOutput(List parameters) { return parameters.stream() .filter(pipelineRunParameter -> pipelineRunParameter.getType().contains(OUTPUT_TYPE)) .map(pipelineRunParameter -> @@ -284,7 +284,7 @@ List createTesOutput(List parameters) { .build()).collect(Collectors.toList()); } - TesResources createTesResources(PipelineRun run) { + private TesResources createTesResources(PipelineRun run) { return TesResources.builder() .preemptible(run.getInstance().getSpot()) .diskGb(new Double(run.getInstance().getNodeDisk())) @@ -294,11 +294,11 @@ TesResources createTesResources(PipelineRun run) { .build(); } - List getPipelineRunZone(PipelineRun run) { + private List getPipelineRunZone(PipelineRun run) { return Collections.singletonList(cloudPipelineAPIClient.loadRegion(run.getInstance().getCloudRegionId()).getRegionCode()); } - List createListExecutor(PipelineRun run) { + private List createListExecutor(PipelineRun run) { return ListUtils.emptyIfNull(Collections.singletonList(TesExecutor.builder() .command(ListUtils.emptyIfNull(Arrays.asList(run.getActualCmd().split(SEPARATOR)))) .env(run.getEnvVars()) @@ -306,7 +306,7 @@ List createListExecutor(PipelineRun run) { .build())); } - List createTesTaskLog(final Long runId) { + private List createTesTaskLog(final Long runId) { List runLogList = ListUtils.emptyIfNull(cloudPipelineAPIClient.getRunLog(runId)); List tesExecutorLogList = Collections.singletonList(TesExecutorLog.builder() .stdout(runLogList.stream() @@ -318,7 +318,7 @@ List createTesTaskLog(final Long runId) { .build()); } - InstanceType getInstanceType(PipelineRun run) { + private InstanceType getInstanceType(PipelineRun run) { Long regionId = run.getInstance().getCloudRegionId(); Boolean spot = run.getInstance().getSpot(); Tool pipeLineTool = loadToolByTesImage(run.getDockerImage()); From 5a543aeafceab16034fe2ffba3907550f276312c Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 24 Sep 2019 16:38:40 +0300 Subject: [PATCH 130/194] merged with tes-support --- .../pipeline/tesadapter/service/TaskMapperTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index b89e220794..048a878164 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -82,7 +82,6 @@ class TaskMapperTest { private static final String STUBBED_IMAGE = "cp-docker-registry.default.svc.cluster.local:31443/library/centos:latest"; private static final String INPUT = "input"; private static final String OUTPUT = "output"; - private static final Double KIB_TO_GIB = 0.00000095367432; private static final String STUBBED_NAME_PIPELINE_PARAMETER = "pipelineRunParameter"; private static final String STUBBED_VALUE_PIPELINE_PARAMETER = "www.url.ru"; private static final String OUTPUT_LOG_STRING_FORMAT = "%s - %s - %s"; @@ -132,10 +131,7 @@ public static void setUpAll() { when(cloudPipelineAPIClient.loadRegion(anyLong())).thenReturn(awsRegion); when(cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(TOOL_ID, 1l, true)).thenReturn(getAllowedInstanceAndPriceTypes(run)); - Tool tool = new Tool(); - tool.setId(TOOL_ID); - tool.setName(STUBBED_IMAGE); - when(cloudPipelineAPIClient.loadTool(anyString())).thenReturn(tool); + } @BeforeEach @@ -377,6 +373,10 @@ private static InstanceType getInstanceWithNameRamAndCpu(String instanceName, Do @ParameterizedTest @MethodSource("provideTesTaskForTestTaskMapper") public void testTaskMapperMapPipelineRunToTesTask(PipelineRun run, TaskView view, TesTask tesTaskExpected) { + Tool tool = new Tool(); + tool.setId(TOOL_ID); + tool.setName(STUBBED_IMAGE); + when(cloudPipelineAPIClient.loadTool(anyString())).thenReturn(tool); TesTask tesTaskActual = taskMapper.mapToTesTask(run, view); assertEquals(tesTaskExpected, tesTaskActual); } From 393cb1e9280a28c3fbe8417219b46017fa6f1817 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 24 Sep 2019 17:35:56 +0300 Subject: [PATCH 131/194] refactoring TasMapper --- .../java/com/epam/pipeline/tesadapter/service/TaskMapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index e75b4cee3e..59cab760d5 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -193,7 +193,7 @@ private Double calculateInstanceCoef(InstanceType instanceType, Double ramGb, Lo - ramGb) / ramGb + Math.abs((double) (instanceType.getVCPU() - cpuCores)) / cpuCores; } - Double convertMemoryUnitTypeToGiB(String memoryUnit) { + private Double convertMemoryUnitTypeToGiB(String memoryUnit) { if (memoryUnit != null) { if (memoryUnit.equalsIgnoreCase(PipelineDiskMemoryTypes.KIB.getValue())) { return KIB_TO_GIB; @@ -210,7 +210,7 @@ Double convertMemoryUnitTypeToGiB(String memoryUnit) { return GIB_TO_GIB; } - public TesTask mapToTesTask(PipelineRun run, TaskView view) { + TesTask mapToTesTask(PipelineRun run, TaskView view) { final TesTask.TesTaskBuilder tesTask = TesTask.builder() .id(String.valueOf(run.getId())) .state(createTesState(run)); From 0a0b3dc1c90886e2ceae9e6b33328b2779ac69a6 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 24 Sep 2019 17:45:13 +0300 Subject: [PATCH 132/194] refactoring TasMapper --- .../java/com/epam/pipeline/tesadapter/service/TaskMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 59cab760d5..4198d7defa 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -210,7 +210,7 @@ private Double convertMemoryUnitTypeToGiB(String memoryUnit) { return GIB_TO_GIB; } - TesTask mapToTesTask(PipelineRun run, TaskView view) { + public TesTask mapToTesTask(PipelineRun run, TaskView view) { final TesTask.TesTaskBuilder tesTask = TesTask.builder() .id(String.valueOf(run.getId())) .state(createTesState(run)); From f5aaf5a5b77fc6ca55be8340768a5d46fd097e4b Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Tue, 24 Sep 2019 17:52:37 +0300 Subject: [PATCH 133/194] make mapToPipelineStart public - since it is API for TaskMapper class --- .../java/com/epam/pipeline/tesadapter/service/TaskMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 4198d7defa..5f588cc33a 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -91,7 +91,7 @@ public TaskMapper(@Value("${cloud.pipeline.hddSize}") Integer hddSize, this.defaultRegion = defaultRegion; } - PipelineStart mapToPipelineStart(TesTask tesTask) { + public PipelineStart mapToPipelineStart(TesTask tesTask) { Assert.notNull(tesTask, messageHelper.getMessage( MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, tesTask)); PipelineStart pipelineStart = new PipelineStart(); From bac00f28f0d5efb6ef741c0470b758607b04c0d5 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 24 Sep 2019 19:14:25 +0300 Subject: [PATCH 134/194] implemented TesTokenInterceptor, configured corresponded TesTokenHolder and CloudPipelineAPIClient in order to use them correctly with last one --- .../configuration/RestConfiguration.java | 22 +++++++ .../tesadapter/entity/TesTokenHolder.java | 8 +++ .../service/CloudPipelineAPIClient.java | 59 ++++++++++++------ .../service/TesTokenInterceptor.java | 61 +++++++++++++++++++ 4 files changed, 130 insertions(+), 20 deletions(-) create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTokenHolder.java create mode 100644 tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTokenInterceptor.java diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java index fa54d6c4b2..c24a1fd29e 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java @@ -1,10 +1,16 @@ package com.epam.pipeline.tesadapter.configuration; +import com.epam.pipeline.tesadapter.entity.TesTokenHolder; +import com.epam.pipeline.tesadapter.service.TesTokenInterceptor; import org.springframework.boot.autoconfigure.http.HttpMessageConverters; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Scope; +import org.springframework.context.annotation.ScopedProxyMode; import org.springframework.http.converter.HttpMessageConverter; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.web.context.WebApplicationContext; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; import java.util.Collections; @@ -17,4 +23,20 @@ public HttpMessageConverters customConverters() { HttpMessageConverter addition = new MappingJackson2HttpMessageConverter(); return new HttpMessageConverters(false, Collections.singletonList(addition)); } + + @Override + public void addInterceptors(InterceptorRegistry registry) { + registry.addInterceptor(getTesTokenInterceptor()); + } + + @Bean + public TesTokenInterceptor getTesTokenInterceptor() { + return new TesTokenInterceptor(getTesTokenHolder()); + } + + @Bean + @Scope(value = WebApplicationContext.SCOPE_REQUEST, proxyMode = ScopedProxyMode.TARGET_CLASS) + public TesTokenHolder getTesTokenHolder() { + return new TesTokenHolder(); + } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTokenHolder.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTokenHolder.java new file mode 100644 index 0000000000..009c18d5bb --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesTokenHolder.java @@ -0,0 +1,8 @@ +package com.epam.pipeline.tesadapter.entity; + +import lombok.Data; + +@Data +public class TesTokenHolder { + private String token; +} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index 3ab0805e26..fd051c184b 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -11,10 +11,12 @@ import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.entity.region.AbstractCloudRegion; import com.epam.pipeline.rest.PagedResult; +import com.epam.pipeline.tesadapter.entity.TesTokenHolder; import com.epam.pipeline.utils.QueryUtils; import com.epam.pipeline.vo.PagingRunFilterExpressionVO; import com.epam.pipeline.vo.PagingRunFilterVO; import com.epam.pipeline.vo.RunStatusVO; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -22,62 +24,79 @@ @Service public class CloudPipelineAPIClient { - private CloudPipelineAPI cloudPipelineAPI; + private TesTokenHolder tesTokenHolder; - public CloudPipelineAPIClient(@Value("${cloud.pipeline.host}") String cloudPipelineHostUrl, - @Value("${cloud.pipeline.token}") String cloudPipelineToken) { - this.cloudPipelineAPI = - new CloudPipelineApiBuilder(0, 0, cloudPipelineHostUrl, cloudPipelineToken) - .buildClient(); + @Value("${cloud.pipeline.host}") + private String cloudPipelineHostUrl; + + @Autowired + public CloudPipelineAPIClient(TesTokenHolder tesTokenHolder) { + this.tesTokenHolder = tesTokenHolder; + } + + private CloudPipelineAPI buildCloudPipelineAPI(String apiHost, String token) { + return new CloudPipelineApiBuilder(0, 0, apiHost, token) + .buildClient(); } public PipelineRun runPipeline(PipelineStart runVo) { - return QueryUtils.execute(cloudPipelineAPI.runPipeline(runVo)); + return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) + .runPipeline(runVo)); } public PipelineRun loadPipelineRun(final Long pipelineRunId) { - return QueryUtils.execute(cloudPipelineAPI.loadPipelineRun(pipelineRunId)); + return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) + .loadPipelineRun(pipelineRunId)); } public PipelineRun updateRunStatus(final Long pipelineRunId, RunStatusVO statusUpdate) { - return QueryUtils.execute(cloudPipelineAPI.updateRunStatus(pipelineRunId, statusUpdate)); + return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) + .updateRunStatus(pipelineRunId, statusUpdate)); } public AllowedInstanceAndPriceTypes loadAllowedInstanceAndPriceTypes(final Long toolId, final Long regionId, final Boolean spot) { - return QueryUtils.execute(cloudPipelineAPI.loadAllowedInstanceAndPriceTypes(toolId, regionId, spot)); + return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) + .loadAllowedInstanceAndPriceTypes(toolId, regionId, spot)); } public Tool loadTool(String image) { - return QueryUtils.execute(cloudPipelineAPI.loadTool(null, image)); + return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) + .loadTool(null, image)); } public List loadAllRegions() { - return QueryUtils.execute(cloudPipelineAPI.loadAllRegions()); + return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) + .loadAllRegions()); } public List loadAllDataStorages() { - return QueryUtils.execute(cloudPipelineAPI.loadAllDataStorages()); + return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) + .loadAllDataStorages()); } public List getRunLog(final Long runId) { - return QueryUtils.execute(cloudPipelineAPI.loadLogs(runId)); + return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) + .loadLogs(runId)); } - public List loadPipelineTasks(final Long id) { - return QueryUtils.execute(cloudPipelineAPI.loadPipelineTasks(id)); + return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) + .loadPipelineTasks(id)); } public PagedResult> searchRuns(PagingRunFilterExpressionVO filterVO) { - return QueryUtils.execute(cloudPipelineAPI.searchRuns(filterVO)); + return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) + .searchRuns(filterVO)); } public PagedResult> filterRuns(PagingRunFilterVO filterVO, Boolean loadStorageLinks) { - return QueryUtils.execute(cloudPipelineAPI.filterRuns(filterVO, loadStorageLinks)); + return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) + .filterRuns(filterVO, loadStorageLinks)); } - public AbstractCloudRegion loadRegion(final Long regionId){ - return QueryUtils.execute(cloudPipelineAPI.loadRegion(regionId)); + public AbstractCloudRegion loadRegion(final Long regionId) { + return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) + .loadRegion(regionId)); } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTokenInterceptor.java new file mode 100644 index 0000000000..6a5b5b5e9e --- /dev/null +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTokenInterceptor.java @@ -0,0 +1,61 @@ +package com.epam.pipeline.tesadapter.service; + +import com.epam.pipeline.tesadapter.entity.TesTokenHolder; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpHeaders; +import org.springframework.web.servlet.HandlerInterceptor; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.Arrays; +import java.util.Optional; + +@Slf4j +@SuppressWarnings("unused") +public class TesTokenInterceptor implements HandlerInterceptor { + private static final String HTTP_AUTH_COOKIE = "HttpAuthorization"; + + private TesTokenHolder tesTokenHolder; + + @Value("${cloud.pipeline.token}") + private String defaultPipelineToken; + + @Autowired + public TesTokenInterceptor(TesTokenHolder tesTokenHolder) { + this.tesTokenHolder = tesTokenHolder; + } + + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { + if (checkRequestForToken(request).isPresent()) { + tesTokenHolder.setToken(checkRequestForToken(request).get()); + return true; + } else if (checkClientHostAddress(request)) { + tesTokenHolder.setToken(defaultPipelineToken); + return true; + } + response.sendError(HttpServletResponse.SC_FORBIDDEN); + return false; + } + + private Optional checkRequestForToken(HttpServletRequest request) { + if (StringUtils.isNotEmpty(request.getHeader(HttpHeaders.AUTHORIZATION))) { + return Optional.of(request.getHeader(HttpHeaders.AUTHORIZATION)); + } else if (ArrayUtils.isNotEmpty(request.getCookies())) { + return Arrays.stream(request.getCookies()).filter(cookie -> + cookie.getName().equalsIgnoreCase(HTTP_AUTH_COOKIE)) + .map(Cookie::getValue).findFirst(); + } + return Optional.empty(); + } + + private boolean checkClientHostAddress(HttpServletRequest request) { + //stub + return true; + } +} From efa7066226e86f1cc5710f5b0d65ccd9330fd61a Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 24 Sep 2019 19:18:08 +0300 Subject: [PATCH 135/194] merge with tes-support branch --- .../epam/pipeline/tesadapter/service/TaskMapperTest.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index 048a878164..fbec46ae51 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -8,8 +8,8 @@ import com.epam.pipeline.entity.pipeline.RunLog; import com.epam.pipeline.entity.pipeline.TaskStatus; import com.epam.pipeline.entity.pipeline.Tool; -import com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter; import com.epam.pipeline.entity.pipeline.run.PipelineStart; +import com.epam.pipeline.entity.pipeline.run.parameter.PipelineRunParameter; import com.epam.pipeline.entity.region.AbstractCloudRegion; import com.epam.pipeline.entity.region.AwsRegion; import com.epam.pipeline.tesadapter.common.MessageConstants; @@ -18,8 +18,6 @@ import com.epam.pipeline.tesadapter.entity.PipelineDiskMemoryTypes; import com.epam.pipeline.tesadapter.entity.TaskView; import com.epam.pipeline.tesadapter.entity.TesExecutor; -import com.epam.pipeline.tesadapter.entity.TesInput; -import com.epam.pipeline.tesadapter.entity.TesOutput; import com.epam.pipeline.tesadapter.entity.TesExecutorLog; import com.epam.pipeline.tesadapter.entity.TesInput; import com.epam.pipeline.tesadapter.entity.TesOutput; @@ -50,9 +48,9 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.springframework.util.StringUtils.hasText; @@ -88,7 +86,7 @@ class TaskMapperTest { private static final String TYPE_NODE = "nodeType"; private static final String STUBBED_TEXT_FOR_LOG = "log"; private static final String PIPELINE_TASK_NAME_CONSOLE = "Console"; - private static final String PIPELINE_TASK_NAME_InitializeEnvironment = "InitializeEnvironment"; + private static final String PIPELINE_TASK_NAME_InitializeEnvironment = "InitializeEnvironment"; private static final String ANY_STRING = "any_string"; private static List allowedInstanceTypes; private static final String GIB = "GiB"; From 80462831105e34efc0f7367b014cb5198bd8b223 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 25 Sep 2019 12:42:50 +0300 Subject: [PATCH 136/194] fixed couple mistakes, replaced TesTokenInterceptor to controllers, refactored APIClient class --- .../configuration/RestConfiguration.java | 2 +- .../TesTokenInterceptor.java | 2 +- .../service/CloudPipelineAPIClient.java | 46 ++++++++----------- 3 files changed, 20 insertions(+), 30 deletions(-) rename tes-adapter/src/main/java/com/epam/pipeline/tesadapter/{service => controller}/TesTokenInterceptor.java (97%) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java index c24a1fd29e..db96213f13 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java @@ -1,7 +1,7 @@ package com.epam.pipeline.tesadapter.configuration; import com.epam.pipeline.tesadapter.entity.TesTokenHolder; -import com.epam.pipeline.tesadapter.service.TesTokenInterceptor; +import com.epam.pipeline.tesadapter.controller.TesTokenInterceptor; import org.springframework.boot.autoconfigure.http.HttpMessageConverters; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java similarity index 97% rename from tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTokenInterceptor.java rename to tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index 6a5b5b5e9e..15e78f7b9d 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -1,4 +1,4 @@ -package com.epam.pipeline.tesadapter.service; +package com.epam.pipeline.tesadapter.controller; import com.epam.pipeline.tesadapter.entity.TesTokenHolder; import lombok.extern.slf4j.Slf4j; diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index fd051c184b..3e4f4d8af6 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -26,77 +26,67 @@ public class CloudPipelineAPIClient { private TesTokenHolder tesTokenHolder; - @Value("${cloud.pipeline.host}") private String cloudPipelineHostUrl; @Autowired - public CloudPipelineAPIClient(TesTokenHolder tesTokenHolder) { + public CloudPipelineAPIClient(TesTokenHolder tesTokenHolder, + @Value("${cloud.pipeline.host}") String cloudPipelineHostUrl) { this.tesTokenHolder = tesTokenHolder; + this.cloudPipelineHostUrl = cloudPipelineHostUrl; } - private CloudPipelineAPI buildCloudPipelineAPI(String apiHost, String token) { - return new CloudPipelineApiBuilder(0, 0, apiHost, token) + private CloudPipelineAPI buildCloudPipelineAPI() { + return new CloudPipelineApiBuilder(0, 0, cloudPipelineHostUrl, + tesTokenHolder.getToken()) .buildClient(); } public PipelineRun runPipeline(PipelineStart runVo) { - return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) - .runPipeline(runVo)); + return QueryUtils.execute(buildCloudPipelineAPI().runPipeline(runVo)); } public PipelineRun loadPipelineRun(final Long pipelineRunId) { - return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) - .loadPipelineRun(pipelineRunId)); + return QueryUtils.execute(buildCloudPipelineAPI().loadPipelineRun(pipelineRunId)); } public PipelineRun updateRunStatus(final Long pipelineRunId, RunStatusVO statusUpdate) { - return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) - .updateRunStatus(pipelineRunId, statusUpdate)); + return QueryUtils.execute(buildCloudPipelineAPI().updateRunStatus(pipelineRunId, statusUpdate)); } public AllowedInstanceAndPriceTypes loadAllowedInstanceAndPriceTypes(final Long toolId, final Long regionId, final Boolean spot) { - return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) - .loadAllowedInstanceAndPriceTypes(toolId, regionId, spot)); + return QueryUtils.execute(buildCloudPipelineAPI().loadAllowedInstanceAndPriceTypes(toolId, regionId, spot)); } public Tool loadTool(String image) { - return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) - .loadTool(null, image)); + return QueryUtils.execute(buildCloudPipelineAPI().loadTool(null, image)); } public List loadAllRegions() { - return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) - .loadAllRegions()); + return QueryUtils.execute(buildCloudPipelineAPI().loadAllRegions()); } public List loadAllDataStorages() { - return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) - .loadAllDataStorages()); + return QueryUtils.execute(buildCloudPipelineAPI().loadAllDataStorages()); } public List getRunLog(final Long runId) { - return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) - .loadLogs(runId)); + return QueryUtils.execute(buildCloudPipelineAPI().loadLogs(runId)); } public List loadPipelineTasks(final Long id) { - return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) - .loadPipelineTasks(id)); + return QueryUtils.execute(buildCloudPipelineAPI().loadPipelineTasks(id)); } public PagedResult> searchRuns(PagingRunFilterExpressionVO filterVO) { - return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) - .searchRuns(filterVO)); + return QueryUtils.execute(buildCloudPipelineAPI().searchRuns(filterVO)); } public PagedResult> filterRuns(PagingRunFilterVO filterVO, Boolean loadStorageLinks) { - return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) - .filterRuns(filterVO, loadStorageLinks)); + return QueryUtils.execute(buildCloudPipelineAPI().filterRuns(filterVO, loadStorageLinks)); } public AbstractCloudRegion loadRegion(final Long regionId) { - return QueryUtils.execute(buildCloudPipelineAPI(cloudPipelineHostUrl, tesTokenHolder.getToken()) - .loadRegion(regionId)); + return QueryUtils.execute(buildCloudPipelineAPI().loadRegion(regionId)); } } From caaddb7ba3c2c9a63ad9a79c16f278317fc3c364 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 25 Sep 2019 17:55:08 +0300 Subject: [PATCH 137/194] fixed error code to SC_UNAUTHORIZED in response --- .../pipeline/tesadapter/controller/TesTokenInterceptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index 15e78f7b9d..26c157d6fe 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -39,7 +39,7 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons tesTokenHolder.setToken(defaultPipelineToken); return true; } - response.sendError(HttpServletResponse.SC_FORBIDDEN); + response.sendError(HttpServletResponse.SC_UNAUTHORIZED); return false; } From 864b87faa27ad00ee349f8fc6b9c4ca8c31f9c43 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 25 Sep 2019 19:57:47 +0300 Subject: [PATCH 138/194] Implement logic of checkClientHostAddress in Interceptor class --- tes-adapter/build.gradle | 1 + .../tesadapter/controller/TesTokenInterceptor.java | 8 ++++++-- tes-adapter/src/main/resources/application.properties | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tes-adapter/build.gradle b/tes-adapter/build.gradle index 59f22b1277..696a985a4e 100644 --- a/tes-adapter/build.gradle +++ b/tes-adapter/build.gradle @@ -39,6 +39,7 @@ dependencies { //Spring compile('org.springframework.boot:spring-boot-starter-web') + compile group: 'org.springframework.security', name: 'spring-security-web', version: '5.1.6.RELEASE' //Swagger compile('io.springfox:springfox-swagger2:2.9.2') diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index 26c157d6fe..c66416348e 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -7,6 +7,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpHeaders; +import org.springframework.security.web.util.matcher.IpAddressMatcher; import org.springframework.web.servlet.HandlerInterceptor; import javax.servlet.http.Cookie; @@ -25,6 +26,9 @@ public class TesTokenInterceptor implements HandlerInterceptor { @Value("${cloud.pipeline.token}") private String defaultPipelineToken; + @Value("${security.allowed.client.ip.range}") + private String ipRange; + @Autowired public TesTokenInterceptor(TesTokenHolder tesTokenHolder) { this.tesTokenHolder = tesTokenHolder; @@ -55,7 +59,7 @@ private Optional checkRequestForToken(HttpServletRequest request) { } private boolean checkClientHostAddress(HttpServletRequest request) { - //stub - return true; + IpAddressMatcher ipAddressMatcher = new IpAddressMatcher(ipRange); + return ipAddressMatcher.matches(request); } } diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index dec9bd3638..1ead970c63 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -8,4 +8,5 @@ cloud.pipeline.preemtible=true cloud.pipeline.region=eu-central-1 cloud.pipeline.service.name=CloudPipeline cloud.pipeline.doc=https://epam.github.io/cloud-pipeline/ +security.allowed.client.ip.range=192.168.1.0/24 From 333df9eda28c4c43c6b7712325fe13032f2c5374 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 26 Sep 2019 00:10:22 +0300 Subject: [PATCH 139/194] added simple test for preHandle method, need more tests --- .../tesadapter/app/TesAdapterControllerTest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index a33f9d5e38..9954fdd369 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -59,6 +59,9 @@ public class TesAdapterControllerTest { @Value("${cloud.pipeline.doc}") private String doc; + @Value("${cloud.pipeline.token}") + private String defaultPipelineToken; + @Autowired private MessageHelper messageHelper; @@ -152,4 +155,12 @@ void serviceInfoRequestShouldReturnCurrentServiceState() throws Exception { .andDo(print()).andExpect(status().isOk()) .andExpect(content().json(new ObjectMapper().writeValueAsString(tesServiceInfo))); } + + @Test + void preHandleMethodShouldCheckAuthorizationContext() throws Exception { + this.mockMvc.perform(get("/v1/tasks/service-info") + .header("Authorization", defaultPipelineToken).contentType(JSON_CONTENT_TYPE)) + .andDo(print()).andExpect(status().isOk()) + .andExpect(content().json(new ObjectMapper().writeValueAsString(tesServiceInfo))); + } } From 5ccabb2cfa47b2bdc61fcb9e65659324b90f7c05 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 26 Sep 2019 14:34:44 +0300 Subject: [PATCH 140/194] added more tests for preHandle method. added headers for rest tests in Test Web-layer class --- .../controller/TesTokenInterceptor.java | 2 +- .../src/main/resources/application.properties | 3 +- .../app/TesAdapterControllerTest.java | 67 ++++++++++++++++--- 3 files changed, 62 insertions(+), 10 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index 26c157d6fe..0e576bd4af 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -56,6 +56,6 @@ private Optional checkRequestForToken(HttpServletRequest request) { private boolean checkClientHostAddress(HttpServletRequest request) { //stub - return true; + return false; } } diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index dec9bd3638..474cbfd025 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -8,4 +8,5 @@ cloud.pipeline.preemtible=true cloud.pipeline.region=eu-central-1 cloud.pipeline.service.name=CloudPipeline cloud.pipeline.doc=https://epam.github.io/cloud-pipeline/ - +allowed.client.ip=192.168.1.1 +wrong.client.ip=10.0.0.0 \ No newline at end of file diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 9954fdd369..09dc698758 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -20,8 +20,10 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.HttpHeaders; import org.springframework.test.web.servlet.MockMvc; +import javax.servlet.http.Cookie; import java.util.ArrayList; import java.util.Collections; @@ -52,6 +54,15 @@ public class TesAdapterControllerTest { private static final String STUBBED_SUBMIT_JSON_RESPONSE = "{\"id\":\"5\"}"; private static final String CANCEL_REQUEST_DESCRIPTION = "uri=/v1/tasks/%20:cancel;client=127.0.0.1"; private static final String JSON_CONTENT_TYPE = "application/json"; + private static final String HTTP_AUTH_COOKIE = "HttpAuthorization"; + private static final String WRONG_TOKEN = "wrongPipelineToken"; + private static final String GET_SERVICE_INFO = "/v1/tasks/service-info"; + + @Value("${allowed.client.ip}") + private String ipInRange; + + @Value("${wrong.client.ip}") + private String ipOutOfRange; @Value("${cloud.pipeline.service.name}") private String nameOfService; @@ -93,7 +104,9 @@ void setUp() { void submitTesTaskWhenRequestingTesTaskBodyAndReturnId() throws Exception { tesCreateTaskResponse.setId(DEFAULT_TASK_ID); when(tesTaskService.submitTesTask(any(TesTask.class))).thenReturn(tesCreateTaskResponse); - this.mockMvc.perform(post("/v1/tasks").contentType(JSON_CONTENT_TYPE) + this.mockMvc.perform(post("/v1/tasks") + .header(HttpHeaders.AUTHORIZATION, defaultPipelineToken) + .contentType(JSON_CONTENT_TYPE) .content(STUBBED_SUBMIT_JSON_REQUEST)) .andDo(print()).andExpect(status().isOk()).andExpect(content() .json(new ObjectMapper().writeValueAsString(tesCreateTaskResponse))); @@ -104,6 +117,7 @@ void expectIllegalArgExceptionWhenRunSubmitTesTasWithNullId() throws Exception { tesTask.setExecutors(null); when(tesTaskService.submitTesTask(tesTask)).thenThrow(new IllegalArgumentException()); this.mockMvc.perform(post("/v1/tasks") + .header(HttpHeaders.AUTHORIZATION, defaultPipelineToken) .contentType(JSON_CONTENT_TYPE) .content(STUBBED_SUBMIT_JSON_REQUEST)) .andDo(print()).andExpect(status().isInternalServerError()); @@ -112,7 +126,8 @@ void expectIllegalArgExceptionWhenRunSubmitTesTasWithNullId() throws Exception { @Test void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { when(tesTaskService.cancelTesTask(DEFAULT_TASK_ID)).thenReturn(new TesCancelTaskResponse()); - this.mockMvc.perform(post("/v1/tasks/{id}:cancel", DEFAULT_TASK_ID)) + this.mockMvc.perform(post("/v1/tasks/{id}:cancel", DEFAULT_TASK_ID) + .header(HttpHeaders.AUTHORIZATION, defaultPipelineToken)) .andDo(print()).andExpect(status().isOk()); } @@ -120,7 +135,8 @@ void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { void expectIllegalStateExceptionWhenRunCancelTesTaskWithWrongId() throws Exception { when(tesTaskService.cancelTesTask(EMPTY_INPUT)).thenThrow(new IllegalStateException(messageHelper .getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, "taskId"))); - this.mockMvc.perform(post("/v1/tasks/{id}:cancel", EMPTY_INPUT)) + this.mockMvc.perform(post("/v1/tasks/{id}:cancel", EMPTY_INPUT) + .header(HttpHeaders.AUTHORIZATION, defaultPipelineToken)) .andDo(print()) .andExpect(status().isInternalServerError()) .andExpect(content().string(containsString(messageHelper @@ -132,7 +148,8 @@ void expectIllegalStateExceptionWhenRunCancelTesTaskWithWrongId() throws Excepti void listTesTaskWhenRequestingReturnTesListTasksResponse() throws Exception { this.mockMvc.perform(get("/v1/tasks?name_prefix={name_prefix}?page_size={page_size}" + "?page_token={page_token}?view={view}", - NAME_PREFIX, PAGE_SIZE, PAGE_TOKEN, DEFAULT_VIEW)) + NAME_PREFIX, PAGE_SIZE, PAGE_TOKEN, DEFAULT_VIEW) + .header(HttpHeaders.AUTHORIZATION, defaultPipelineToken)) .andDo(print()).andExpect(status().isOk()); } @@ -141,7 +158,9 @@ void getTesTaskWhenRequestingReturnTesTaskResponse() throws Exception { tesExecutor.setImage(DEFAULT_IMAGE); tesExecutor.setCommand(Collections.singletonList(DEFAULT_COMMAND)); tesTask.setExecutors(Collections.singletonList(tesExecutor)); - this.mockMvc.perform(get("/v1/tasks/{id}", DEFAULT_TASK_ID).contentType(JSON_CONTENT_TYPE)) + this.mockMvc.perform(get("/v1/tasks/{id}", DEFAULT_TASK_ID) + .header(HttpHeaders.AUTHORIZATION, defaultPipelineToken) + .contentType(JSON_CONTENT_TYPE)) .andDo(print()).andExpect(status().isOk()).andExpect(content() .json(new ObjectMapper().writeValueAsString(tesTask))); } @@ -151,16 +170,48 @@ void serviceInfoRequestShouldReturnCurrentServiceState() throws Exception { tesServiceInfo.setName(nameOfService); tesServiceInfo.setDoc(doc); tesServiceInfo.setStorage(new ArrayList<>()); - this.mockMvc.perform(get("/v1/tasks/service-info").contentType(JSON_CONTENT_TYPE)) + this.mockMvc.perform(get("/v1/tasks/service-info") + .header(HttpHeaders.AUTHORIZATION, defaultPipelineToken) + .contentType(JSON_CONTENT_TYPE)) .andDo(print()).andExpect(status().isOk()) .andExpect(content().json(new ObjectMapper().writeValueAsString(tesServiceInfo))); } @Test void preHandleMethodShouldCheckAuthorizationContext() throws Exception { - this.mockMvc.perform(get("/v1/tasks/service-info") - .header("Authorization", defaultPipelineToken).contentType(JSON_CONTENT_TYPE)) + this.mockMvc.perform(get(GET_SERVICE_INFO) + .header(HttpHeaders.AUTHORIZATION, defaultPipelineToken).contentType(JSON_CONTENT_TYPE)) .andDo(print()).andExpect(status().isOk()) .andExpect(content().json(new ObjectMapper().writeValueAsString(tesServiceInfo))); + + this.mockMvc.perform(get(GET_SERVICE_INFO) + .cookie(new Cookie(HTTP_AUTH_COOKIE, defaultPipelineToken)).contentType(JSON_CONTENT_TYPE)) + .andDo(print()).andExpect(status().isOk()) + .andExpect(content().json(new ObjectMapper().writeValueAsString(tesServiceInfo))); + + this.mockMvc.perform(get(GET_SERVICE_INFO) + .contentType(JSON_CONTENT_TYPE)) + .andDo(print()).andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(GET_SERVICE_INFO) + .contentType(JSON_CONTENT_TYPE)) + .andDo(print()).andExpect(status().isUnauthorized()); + +// this.mockMvc.perform(get(GET_SERVICE_INFO) +// .with(request -> { +// request.setRemoteAddr(IP_IN_RANGE); +// return request; +// }) +// .contentType(JSON_CONTENT_TYPE)) +// .andDo(print()).andExpect(status().isOk()) +// .andExpect(content().json(new ObjectMapper().writeValueAsString(tesServiceInfo))); +// +// this.mockMvc.perform(get(GET_SERVICE_INFO) +// .with(request -> { +// request.setRemoteAddr(IP_OUT_OF_RANGE); +// return request; +// }) +// .contentType(JSON_CONTENT_TYPE)) +// .andDo(print()).andExpect(status().isUnauthorized()); } } From f33eb67a7f260a3a34332f2a114f84eff4aa353d Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 26 Sep 2019 14:44:51 +0300 Subject: [PATCH 141/194] couple clean-code in preHandle Test --- .../pipeline/tesadapter/app/TesAdapterControllerTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 09dc698758..735744c67c 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -193,10 +193,6 @@ void preHandleMethodShouldCheckAuthorizationContext() throws Exception { .contentType(JSON_CONTENT_TYPE)) .andDo(print()).andExpect(status().isUnauthorized()); - this.mockMvc.perform(get(GET_SERVICE_INFO) - .contentType(JSON_CONTENT_TYPE)) - .andDo(print()).andExpect(status().isUnauthorized()); - // this.mockMvc.perform(get(GET_SERVICE_INFO) // .with(request -> { // request.setRemoteAddr(IP_IN_RANGE); From 7015c8a5e0593b0229e0e10349fa6b97080bd470 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Thu, 26 Sep 2019 15:32:16 +0300 Subject: [PATCH 142/194] refactoring logic of checkClientHostAddress in Interceptor class. added processing ipRange on null --- .../pipeline/tesadapter/controller/TesTokenInterceptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index c66416348e..6b09f39ffe 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -26,7 +26,7 @@ public class TesTokenInterceptor implements HandlerInterceptor { @Value("${cloud.pipeline.token}") private String defaultPipelineToken; - @Value("${security.allowed.client.ip.range}") + @Value("${security.allowed.client.ip.range:0.0.0.0/0}") private String ipRange; @Autowired From 3018af6f9399451d417f127b752dba0c4e77262b Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 26 Sep 2019 15:32:52 +0300 Subject: [PATCH 143/194] refactored TesAdapterControllerTest class, created test-application.properties file, + added one more PMD suppress rule --- .../src/main/resources/application.properties | 4 +--- .../app/TesAdapterControllerTest.java | 18 ++++++++---------- .../test/resources/test-application.properties | 5 +++++ 3 files changed, 14 insertions(+), 13 deletions(-) create mode 100644 tes-adapter/src/test/resources/test-application.properties diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index 474cbfd025..cab47476d8 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -7,6 +7,4 @@ cloud.pipeline.hddSize=50 cloud.pipeline.preemtible=true cloud.pipeline.region=eu-central-1 cloud.pipeline.service.name=CloudPipeline -cloud.pipeline.doc=https://epam.github.io/cloud-pipeline/ -allowed.client.ip=192.168.1.1 -wrong.client.ip=10.0.0.0 \ No newline at end of file +cloud.pipeline.doc=https://epam.github.io/cloud-pipeline/ \ No newline at end of file diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 735744c67c..285ac4e2a5 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -21,6 +21,7 @@ import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.http.HttpHeaders; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.web.servlet.MockMvc; import javax.servlet.http.Cookie; @@ -39,7 +40,8 @@ @WebMvcTest(TesAdapterController.class) -@SuppressWarnings({"unused", "PMD.TooManyStaticImports"}) +@SuppressWarnings({"unused", "PMD.TooManyStaticImports", "PMD.AvoidUsingHardCodedIP"}) +@TestPropertySource(locations = "classpath:test-application.properties") public class TesAdapterControllerTest { private static final String DEFAULT_TASK_ID = "5"; private static final String PAGE_TOKEN = "1"; @@ -57,12 +59,8 @@ public class TesAdapterControllerTest { private static final String HTTP_AUTH_COOKIE = "HttpAuthorization"; private static final String WRONG_TOKEN = "wrongPipelineToken"; private static final String GET_SERVICE_INFO = "/v1/tasks/service-info"; - - @Value("${allowed.client.ip}") - private String ipInRange; - - @Value("${wrong.client.ip}") - private String ipOutOfRange; + private static final String IP_IN_RANGE = "192.168.1.1"; + private static final String IP_OUT_OF_RANGE = "10.0.0.0"; @Value("${cloud.pipeline.service.name}") private String nameOfService; @@ -98,6 +96,9 @@ void setUp() { .thenReturn(mock(TesListTasksResponse.class)); when(tesTaskService.getServiceInfo()).thenReturn(tesServiceInfo); when(tesTaskService.getTesTask(DEFAULT_TASK_ID, DEFAULT_VIEW)).thenReturn(tesTask); + tesServiceInfo.setName(nameOfService); + tesServiceInfo.setDoc(doc); + tesServiceInfo.setStorage(new ArrayList<>()); } @Test @@ -167,9 +168,6 @@ void getTesTaskWhenRequestingReturnTesTaskResponse() throws Exception { @Test void serviceInfoRequestShouldReturnCurrentServiceState() throws Exception { - tesServiceInfo.setName(nameOfService); - tesServiceInfo.setDoc(doc); - tesServiceInfo.setStorage(new ArrayList<>()); this.mockMvc.perform(get("/v1/tasks/service-info") .header(HttpHeaders.AUTHORIZATION, defaultPipelineToken) .contentType(JSON_CONTENT_TYPE)) diff --git a/tes-adapter/src/test/resources/test-application.properties b/tes-adapter/src/test/resources/test-application.properties new file mode 100644 index 0000000000..9b7bd9e0e2 --- /dev/null +++ b/tes-adapter/src/test/resources/test-application.properties @@ -0,0 +1,5 @@ +server.port=8080 +cloud.pipeline.host=${API} +cloud.pipeline.token=${API_TOKEN} +cloud.pipeline.service.name=CloudPipeline +cloud.pipeline.doc=https://epam.github.io/cloud-pipeline/ \ No newline at end of file From 06832c3526c6e8229c7c13ea3464a7cd8ddbe3eb Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Thu, 26 Sep 2019 17:05:10 +0300 Subject: [PATCH 144/194] refactored default ip range --- .../pipeline/tesadapter/controller/TesTokenInterceptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index 6b09f39ffe..65715475d5 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -26,7 +26,7 @@ public class TesTokenInterceptor implements HandlerInterceptor { @Value("${cloud.pipeline.token}") private String defaultPipelineToken; - @Value("${security.allowed.client.ip.range:0.0.0.0/0}") + @Value("${security.allowed.client.ip.range:0.0.0.1/32}") private String ipRange; @Autowired From 8a47649da2cae5077f3535c68c878f7f424bd5a0 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Thu, 26 Sep 2019 18:05:06 +0300 Subject: [PATCH 145/194] removed couple unnecessary tests for this PR --- .../app/TesAdapterControllerTest.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 285ac4e2a5..413f48a79d 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -190,22 +190,5 @@ void preHandleMethodShouldCheckAuthorizationContext() throws Exception { this.mockMvc.perform(get(GET_SERVICE_INFO) .contentType(JSON_CONTENT_TYPE)) .andDo(print()).andExpect(status().isUnauthorized()); - -// this.mockMvc.perform(get(GET_SERVICE_INFO) -// .with(request -> { -// request.setRemoteAddr(IP_IN_RANGE); -// return request; -// }) -// .contentType(JSON_CONTENT_TYPE)) -// .andDo(print()).andExpect(status().isOk()) -// .andExpect(content().json(new ObjectMapper().writeValueAsString(tesServiceInfo))); -// -// this.mockMvc.perform(get(GET_SERVICE_INFO) -// .with(request -> { -// request.setRemoteAddr(IP_OUT_OF_RANGE); -// return request; -// }) -// .contentType(JSON_CONTENT_TYPE)) -// .andDo(print()).andExpect(status().isUnauthorized()); } } From 64e4dab783c0afb0fb8accd1910d9dc6d3237773 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Thu, 26 Sep 2019 18:43:39 +0300 Subject: [PATCH 146/194] added check ipRange & default token is not null --- .../controller/TesTokenInterceptor.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index 65715475d5..ace8621dd0 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -4,6 +4,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.util.Strings; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpHeaders; @@ -20,13 +21,14 @@ @SuppressWarnings("unused") public class TesTokenInterceptor implements HandlerInterceptor { private static final String HTTP_AUTH_COOKIE = "HttpAuthorization"; + private static final String TOKEN_NOT_FOUND = "Default token not found"; private TesTokenHolder tesTokenHolder; @Value("${cloud.pipeline.token}") private String defaultPipelineToken; - @Value("${security.allowed.client.ip.range:0.0.0.1/32}") + @Value("${security.allowed.client.ip.range}") private String ipRange; @Autowired @@ -40,8 +42,12 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons tesTokenHolder.setToken(checkRequestForToken(request).get()); return true; } else if (checkClientHostAddress(request)) { - tesTokenHolder.setToken(defaultPipelineToken); - return true; + if(Strings.isNotEmpty(defaultPipelineToken)) { + tesTokenHolder.setToken(defaultPipelineToken); + return true; + } else { + response.sendError(HttpServletResponse.SC_UNAUTHORIZED, TOKEN_NOT_FOUND); + } } response.sendError(HttpServletResponse.SC_UNAUTHORIZED); return false; @@ -59,7 +65,11 @@ private Optional checkRequestForToken(HttpServletRequest request) { } private boolean checkClientHostAddress(HttpServletRequest request) { - IpAddressMatcher ipAddressMatcher = new IpAddressMatcher(ipRange); - return ipAddressMatcher.matches(request); + if (StringUtils.isNotEmpty(ipRange)) { + IpAddressMatcher ipAddressMatcher = new IpAddressMatcher(ipRange); + return ipAddressMatcher.matches(request); + } else { + return false; + } } } From cac5729966cb084df513cc9e0e1db182f19a589c Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Thu, 26 Sep 2019 19:00:55 +0300 Subject: [PATCH 147/194] refactoring checkClientHostAddress --- .../controller/TesTokenInterceptor.java | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index ace8621dd0..03cd161c01 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -31,6 +31,8 @@ public class TesTokenInterceptor implements HandlerInterceptor { @Value("${security.allowed.client.ip.range}") private String ipRange; + private IpAddressMatcher ipAddressMatcher = StringUtils.isNotEmpty(ipRange) ? new IpAddressMatcher(ipRange) : null; + @Autowired public TesTokenInterceptor(TesTokenHolder tesTokenHolder) { this.tesTokenHolder = tesTokenHolder; @@ -41,13 +43,9 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons if (checkRequestForToken(request).isPresent()) { tesTokenHolder.setToken(checkRequestForToken(request).get()); return true; - } else if (checkClientHostAddress(request)) { - if(Strings.isNotEmpty(defaultPipelineToken)) { - tesTokenHolder.setToken(defaultPipelineToken); - return true; - } else { - response.sendError(HttpServletResponse.SC_UNAUTHORIZED, TOKEN_NOT_FOUND); - } + } else if (checkClientHostAddress(request) && Strings.isNotEmpty(defaultPipelineToken)) { + tesTokenHolder.setToken(defaultPipelineToken); + return true; } response.sendError(HttpServletResponse.SC_UNAUTHORIZED); return false; @@ -65,11 +63,6 @@ private Optional checkRequestForToken(HttpServletRequest request) { } private boolean checkClientHostAddress(HttpServletRequest request) { - if (StringUtils.isNotEmpty(ipRange)) { - IpAddressMatcher ipAddressMatcher = new IpAddressMatcher(ipRange); - return ipAddressMatcher.matches(request); - } else { - return false; - } + return ipAddressMatcher != null && ipAddressMatcher.matches(request); } } From 292f5b204724ca996bd0f0c1ce20ae6ca82df004 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Thu, 26 Sep 2019 23:54:10 +0300 Subject: [PATCH 148/194] add test preHandle method which should check AuthorizationContext With IpRange --- .../controller/TesTokenInterceptor.java | 10 ++++++---- .../app/TesAdapterControllerTest.java | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index 03cd161c01..ffa2634324 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -21,7 +21,6 @@ @SuppressWarnings("unused") public class TesTokenInterceptor implements HandlerInterceptor { private static final String HTTP_AUTH_COOKIE = "HttpAuthorization"; - private static final String TOKEN_NOT_FOUND = "Default token not found"; private TesTokenHolder tesTokenHolder; @@ -31,8 +30,6 @@ public class TesTokenInterceptor implements HandlerInterceptor { @Value("${security.allowed.client.ip.range}") private String ipRange; - private IpAddressMatcher ipAddressMatcher = StringUtils.isNotEmpty(ipRange) ? new IpAddressMatcher(ipRange) : null; - @Autowired public TesTokenInterceptor(TesTokenHolder tesTokenHolder) { this.tesTokenHolder = tesTokenHolder; @@ -63,6 +60,11 @@ private Optional checkRequestForToken(HttpServletRequest request) { } private boolean checkClientHostAddress(HttpServletRequest request) { - return ipAddressMatcher != null && ipAddressMatcher.matches(request); + if (StringUtils.isNotEmpty(ipRange)) { + IpAddressMatcher ipAddressMatcher = new IpAddressMatcher(ipRange); + return ipAddressMatcher.matches(request); + } else { + return false; + } } } diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 413f48a79d..cf3fa47492 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -191,4 +191,22 @@ void preHandleMethodShouldCheckAuthorizationContext() throws Exception { .contentType(JSON_CONTENT_TYPE)) .andDo(print()).andExpect(status().isUnauthorized()); } + + @Test + void preHandleMethodShouldCheckAuthorizationContextWithIpRange() throws Exception { + this.mockMvc.perform(get(GET_SERVICE_INFO) + .with(request -> { + request.setRemoteAddr(IP_IN_RANGE); + return request; + }) + .contentType(JSON_CONTENT_TYPE)) + .andDo(print()).andExpect(status().isOk()); + this.mockMvc.perform(get(GET_SERVICE_INFO) + .with(request -> { + request.setRemoteAddr(IP_OUT_OF_RANGE); + return request; + }) + .contentType(JSON_CONTENT_TYPE)) + .andDo(print()).andExpect(status().isUnauthorized()); + } } From bd287cb02aad8b5cd7687731f6b5342cd2ecad6b Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Thu, 26 Sep 2019 23:57:09 +0300 Subject: [PATCH 149/194] add test preHandle method which should check AuthorizationContext With IpRange --- .../pipeline/tesadapter/app/TesAdapterControllerTest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index cf3fa47492..1b6c75b816 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -198,15 +198,13 @@ void preHandleMethodShouldCheckAuthorizationContextWithIpRange() throws Exceptio .with(request -> { request.setRemoteAddr(IP_IN_RANGE); return request; - }) - .contentType(JSON_CONTENT_TYPE)) + })) .andDo(print()).andExpect(status().isOk()); this.mockMvc.perform(get(GET_SERVICE_INFO) .with(request -> { request.setRemoteAddr(IP_OUT_OF_RANGE); return request; - }) - .contentType(JSON_CONTENT_TYPE)) + })) .andDo(print()).andExpect(status().isUnauthorized()); } } From 8f2d96466947432ea74085c59fb1fc61fe9a7773 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 27 Sep 2019 17:05:11 +0300 Subject: [PATCH 150/194] added check for app.properties, added defaults values, for these defaults in order to avoid missing property --- .../service/CloudPipelineAPIClient.java | 8 ++++++++ .../pipeline/tesadapter/service/TaskMapper.java | 15 +++++++++------ .../tesadapter/service/TesTaskServiceImpl.java | 4 ++-- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index 3e4f4d8af6..cc64af12be 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -11,6 +11,8 @@ import com.epam.pipeline.entity.pipeline.run.PipelineStart; import com.epam.pipeline.entity.region.AbstractCloudRegion; import com.epam.pipeline.rest.PagedResult; +import com.epam.pipeline.tesadapter.common.MessageConstants; +import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.entity.TesTokenHolder; import com.epam.pipeline.utils.QueryUtils; import com.epam.pipeline.vo.PagingRunFilterExpressionVO; @@ -19,6 +21,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.util.Assert; import java.util.List; @@ -28,9 +31,14 @@ public class CloudPipelineAPIClient { private String cloudPipelineHostUrl; + private static final String CLOUD_PIPELINE_HOST = "cloudPipelineHost"; + @Autowired public CloudPipelineAPIClient(TesTokenHolder tesTokenHolder, + MessageHelper messageHelper, @Value("${cloud.pipeline.host}") String cloudPipelineHostUrl) { + Assert.hasText(cloudPipelineHostUrl, + messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, CLOUD_PIPELINE_HOST)); this.tesTokenHolder = tesTokenHolder; this.cloudPipelineHostUrl = cloudPipelineHostUrl; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 5f588cc33a..73fa03b4a2 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -64,6 +64,7 @@ public class TaskMapper { private static final String IMAGE = "image"; private static final String EXECUTORS = "executors"; private static final String ZONES = "zones"; + private static final String DEFAULT_REGION = "defaultRegion"; private static final Integer FIRST = 0; private static final String OUTPUT_LOG_STRING_FORMAT = "%s - %s - %s"; private static final String CARRIAGE_RETURN = "\n"; @@ -76,11 +77,11 @@ public class TaskMapper { private static final Double EIB_TO_GIB = 1073741824.0; @Autowired - public TaskMapper(@Value("${cloud.pipeline.hddSize}") Integer hddSize, - @Value("${cloud.pipeline.ramGb}") Double defaultRamGb, - @Value("${cloud.pipeline.cpuCore}") Long defaultCpuCore, - @Value("${cloud.pipeline.preemtible}") Boolean defaultPreemptible, - @Value("${cloud.pipeline.region}") String defaultRegion, + public TaskMapper(@Value("#{'${cloud.pipeline.hddSize}' != '' ? '${cloud.pipeline.hddSize}' : 30}") Integer hddSize, + @Value("#{'${cloud.pipeline.ramGb}' != '' ? '${cloud.pipeline.ramGb}' : 4}") Double defaultRamGb, + @Value("#{'${cloud.pipeline.cpuCore}' != '' ? '${cloud.pipeline.cpuCore}' : 2}") Long defaultCpuCore, + @Value("#{'${cloud.pipeline.preemtible}' != '' ? '${cloud.pipeline.preemtible}' : true}") Boolean defaultPreemptible, + @Value("#{'${cloud.pipeline.region}' != '' ? '${cloud.pipeline.region}' : null}") String defaultRegion, CloudPipelineAPIClient cloudPipelineAPIClient, MessageHelper messageHelper) { this.defaultHddSize = hddSize; this.defaultRamGb = defaultRamGb; @@ -138,7 +139,9 @@ String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { Long toolId = pipelineTool.getId(); Long regionId = getProperRegionIdInCloudRegionsByTesZone(Optional.ofNullable(tesTask.getResources()) .map(TesResources::getZones) - .orElse(Collections.singletonList(defaultRegion))); + .orElseGet(() -> Collections.singletonList(Optional.ofNullable(defaultRegion).orElseThrow(() -> + new IllegalArgumentException(messageHelper + .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, DEFAULT_REGION)))))); Boolean spot = Optional.ofNullable(tesTask.getResources()) .map(TesResources::getPreemptible).orElse(defaultPreemptible); AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = Optional.ofNullable(cloudPipelineAPIClient diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 41b58de393..6d39ef339d 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -33,10 +33,10 @@ @Slf4j @Service public class TesTaskServiceImpl implements TesTaskService { - @Value("${cloud.pipeline.service.name}") + @Value("#{'${cloud.pipeline.service.name}' != '' ? '${cloud.pipeline.service.name}' : 'CloudPipeline'}") private String nameOfService; - @Value("${cloud.pipeline.doc}") + @Value("#{'${cloud.pipeline.doc}' != '' ? '${cloud.pipeline.doc}' : ' '}") private String doc; private final CloudPipelineAPIClient cloudPipelineAPIClient; From 1ecf54b8fcd2c0af69c9a6814e1f367d7cfb0676 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 27 Sep 2019 18:49:46 +0300 Subject: [PATCH 151/194] refactored usage default values in app.properties for TaskMapper nad ServiceImpl classes --- .../tesadapter/service/TaskMapper.java | 31 ++++++++++++------- .../service/TesTaskServiceImpl.java | 14 ++++++--- .../src/main/resources/application.properties | 4 +-- .../service/TesTaskServiceImplTest.java | 9 +++++- 4 files changed, 38 insertions(+), 20 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 73fa03b4a2..c3f2409fa1 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -53,6 +53,12 @@ public class TaskMapper { private final CloudPipelineAPIClient cloudPipelineAPIClient; private static final String SEPARATOR = " "; + private static final Integer DEFAULT_HDD_SIZE = 30; + private static final Double DEFAULT_RAM_GB = 4.0; + private static final Long DEFAULT_CPU_CORE = 2L; + private static final Boolean DEFAULT_PREEMPRIBLE = true; + private static final String DEFAULT_REGION = null; + private static final String EMPTY_STRING = ""; private static final String INPUT_TYPE = "input"; private static final String OUTPUT_TYPE = "output"; private static final String DEFAULT_TYPE = "string"; @@ -64,7 +70,7 @@ public class TaskMapper { private static final String IMAGE = "image"; private static final String EXECUTORS = "executors"; private static final String ZONES = "zones"; - private static final String DEFAULT_REGION = "defaultRegion"; + private static final String REGION = "defaultRegion"; private static final Integer FIRST = 0; private static final String OUTPUT_LOG_STRING_FORMAT = "%s - %s - %s"; private static final String CARRIAGE_RETURN = "\n"; @@ -77,19 +83,20 @@ public class TaskMapper { private static final Double EIB_TO_GIB = 1073741824.0; @Autowired - public TaskMapper(@Value("#{'${cloud.pipeline.hddSize}' != '' ? '${cloud.pipeline.hddSize}' : 30}") Integer hddSize, - @Value("#{'${cloud.pipeline.ramGb}' != '' ? '${cloud.pipeline.ramGb}' : 4}") Double defaultRamGb, - @Value("#{'${cloud.pipeline.cpuCore}' != '' ? '${cloud.pipeline.cpuCore}' : 2}") Long defaultCpuCore, - @Value("#{'${cloud.pipeline.preemtible}' != '' ? '${cloud.pipeline.preemtible}' : true}") Boolean defaultPreemptible, - @Value("#{'${cloud.pipeline.region}' != '' ? '${cloud.pipeline.region}' : null}") String defaultRegion, + public TaskMapper(@Value("${cloud.pipeline.hddSize:}") Integer hddSize, + @Value("${cloud.pipeline.ramGb:}") Double defaultRamGb, + @Value("${cloud.pipeline.cpuCore:}") Long defaultCpuCore, + @Value("${cloud.pipeline.preemtible:}") Boolean defaultPreemptible, + @Value("${cloud.pipeline.region:}") String defaultRegion, CloudPipelineAPIClient cloudPipelineAPIClient, MessageHelper messageHelper) { - this.defaultHddSize = hddSize; - this.defaultRamGb = defaultRamGb; - this.defaultCpuCore = defaultCpuCore; + this.defaultHddSize = hddSize != null ? hddSize : DEFAULT_HDD_SIZE; + this.defaultRamGb = defaultRamGb != null ? defaultRamGb : DEFAULT_RAM_GB; + this.defaultCpuCore = defaultCpuCore != null ? defaultCpuCore : DEFAULT_CPU_CORE; + this.defaultPreemptible = defaultPreemptible != null ? defaultPreemptible : DEFAULT_PREEMPRIBLE; + this.defaultRegion = !defaultRegion.equalsIgnoreCase(EMPTY_STRING) ? defaultRegion : DEFAULT_REGION; this.cloudPipelineAPIClient = cloudPipelineAPIClient; this.messageHelper = messageHelper; - this.defaultPreemptible = defaultPreemptible; - this.defaultRegion = defaultRegion; + } public PipelineStart mapToPipelineStart(TesTask tesTask) { @@ -141,7 +148,7 @@ String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { .map(TesResources::getZones) .orElseGet(() -> Collections.singletonList(Optional.ofNullable(defaultRegion).orElseThrow(() -> new IllegalArgumentException(messageHelper - .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, DEFAULT_REGION)))))); + .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, REGION)))))); Boolean spot = Optional.ofNullable(tesTask.getResources()) .map(TesResources::getPreemptible).orElse(defaultPreemptible); AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = Optional.ofNullable(cloudPipelineAPIClient diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 6d39ef339d..292118997d 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -33,12 +33,9 @@ @Slf4j @Service public class TesTaskServiceImpl implements TesTaskService { - @Value("#{'${cloud.pipeline.service.name}' != '' ? '${cloud.pipeline.service.name}' : 'CloudPipeline'}") - private String nameOfService; - @Value("#{'${cloud.pipeline.doc}' != '' ? '${cloud.pipeline.doc}' : ' '}") + private String nameOfService; private String doc; - private final CloudPipelineAPIClient cloudPipelineAPIClient; private final TaskMapper taskMapper; private final MessageHelper messageHelper; @@ -46,12 +43,19 @@ public class TesTaskServiceImpl implements TesTaskService { private static final String ID = "id"; private static final String NAME_PREFIX = "pod.id"; private static final String DEFAULT_PAGE_TOKEN = "1"; + private static final String EMPTY_STRING = ""; + private static final String DEFAULT_NAME_SERVICE = "CloudPipeline"; + private static final String DEFAULT_DOC = " "; private static final Boolean LOAD_STORAGE_LINKS = true; private static final Long DEFAULT_PAGE_SIZE = 256L; @Autowired - public TesTaskServiceImpl(CloudPipelineAPIClient cloudPipelineAPIClient, TaskMapper taskMapper, + public TesTaskServiceImpl(@Value("${cloud.pipeline.service.name:}") String nameOfService, + @Value("${cloud.pipeline.doc:}") String doc, + CloudPipelineAPIClient cloudPipelineAPIClient, TaskMapper taskMapper, MessageHelper messageHelper) { + this.nameOfService = !nameOfService.equalsIgnoreCase(EMPTY_STRING) ? nameOfService : DEFAULT_NAME_SERVICE; + this.doc = !doc.equalsIgnoreCase(EMPTY_STRING) ? doc : DEFAULT_DOC; this.cloudPipelineAPIClient = cloudPipelineAPIClient; this.taskMapper = taskMapper; this.messageHelper = messageHelper; diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index cab47476d8..726049f8ba 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -1,9 +1,9 @@ server.port=8080 cloud.pipeline.host=${API} cloud.pipeline.token=${API_TOKEN} -cloud.pipeline.ramGb=16 +cloud.pipeline.ramGb=4 cloud.pipeline.cpuCore=2 -cloud.pipeline.hddSize=50 +cloud.pipeline.hddSize=30 cloud.pipeline.preemtible=true cloud.pipeline.region=eu-central-1 cloud.pipeline.service.name=CloudPipeline diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java index 7f261c5e63..6636f66b4f 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java @@ -21,6 +21,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -40,6 +41,12 @@ @ContextConfiguration(classes = {AppConfiguration.class}) @SuppressWarnings({"unused", "PMD.TooManyStaticImports"}) class TesTaskServiceImplTest { + @Value("${cloud.pipeline.service.name}") + private String nameOfService; + + @Value("${cloud.pipeline.doc}") + private String doc; + private TesTaskServiceImpl tesTaskService; @Autowired @@ -79,7 +86,7 @@ public void setUp() { pipelineRun.setId(DEFAULT_PIPELINE_ID); tesTask.setId(DEFAULT_PIPELINE_ID.toString()); pipelineRunList.add(pipelineRun); - tesTaskService = new TesTaskServiceImpl(cloudPipelineAPIClient, taskMapper, messageHelper); + tesTaskService = new TesTaskServiceImpl(nameOfService, doc, cloudPipelineAPIClient, taskMapper, messageHelper); } @Test From 0310bcd1a3136b994a54d999692d2d165cd8473e Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 27 Sep 2019 18:55:29 +0300 Subject: [PATCH 152/194] refactored usage default values -> added StringUtils feature --- .../com/epam/pipeline/tesadapter/service/TaskMapper.java | 4 ++-- .../epam/pipeline/tesadapter/service/TesTaskServiceImpl.java | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index c3f2409fa1..5ed93918f7 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.ListUtils; import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -58,7 +59,6 @@ public class TaskMapper { private static final Long DEFAULT_CPU_CORE = 2L; private static final Boolean DEFAULT_PREEMPRIBLE = true; private static final String DEFAULT_REGION = null; - private static final String EMPTY_STRING = ""; private static final String INPUT_TYPE = "input"; private static final String OUTPUT_TYPE = "output"; private static final String DEFAULT_TYPE = "string"; @@ -93,7 +93,7 @@ public TaskMapper(@Value("${cloud.pipeline.hddSize:}") Integer hddSize, this.defaultRamGb = defaultRamGb != null ? defaultRamGb : DEFAULT_RAM_GB; this.defaultCpuCore = defaultCpuCore != null ? defaultCpuCore : DEFAULT_CPU_CORE; this.defaultPreemptible = defaultPreemptible != null ? defaultPreemptible : DEFAULT_PREEMPRIBLE; - this.defaultRegion = !defaultRegion.equalsIgnoreCase(EMPTY_STRING) ? defaultRegion : DEFAULT_REGION; + this.defaultRegion = StringUtils.isNoneEmpty(defaultRegion) ? defaultRegion : DEFAULT_REGION; this.cloudPipelineAPIClient = cloudPipelineAPIClient; this.messageHelper = messageHelper; diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 292118997d..fecc08a3e6 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -43,7 +43,6 @@ public class TesTaskServiceImpl implements TesTaskService { private static final String ID = "id"; private static final String NAME_PREFIX = "pod.id"; private static final String DEFAULT_PAGE_TOKEN = "1"; - private static final String EMPTY_STRING = ""; private static final String DEFAULT_NAME_SERVICE = "CloudPipeline"; private static final String DEFAULT_DOC = " "; private static final Boolean LOAD_STORAGE_LINKS = true; @@ -54,8 +53,8 @@ public TesTaskServiceImpl(@Value("${cloud.pipeline.service.name:}") String nameO @Value("${cloud.pipeline.doc:}") String doc, CloudPipelineAPIClient cloudPipelineAPIClient, TaskMapper taskMapper, MessageHelper messageHelper) { - this.nameOfService = !nameOfService.equalsIgnoreCase(EMPTY_STRING) ? nameOfService : DEFAULT_NAME_SERVICE; - this.doc = !doc.equalsIgnoreCase(EMPTY_STRING) ? doc : DEFAULT_DOC; + this.nameOfService = StringUtils.isNotEmpty(nameOfService) ? nameOfService : DEFAULT_NAME_SERVICE; + this.doc = StringUtils.isNotEmpty(doc) ? doc : DEFAULT_DOC; this.cloudPipelineAPIClient = cloudPipelineAPIClient; this.taskMapper = taskMapper; this.messageHelper = messageHelper; From e5996c7a6e31e9209f1eb6b0e1f0320de3583b7b Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Sat, 28 Sep 2019 16:33:37 +0300 Subject: [PATCH 153/194] refactored approach to get default values from app.properties, changed ternary ?: to Optional, also added check to defaultRegion param --- .../tesadapter/service/TaskMapper.java | 28 +++++++++---------- .../service/TesTaskServiceImpl.java | 8 +++--- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index 5ed93918f7..b172f0d833 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -58,7 +58,6 @@ public class TaskMapper { private static final Double DEFAULT_RAM_GB = 4.0; private static final Long DEFAULT_CPU_CORE = 2L; private static final Boolean DEFAULT_PREEMPRIBLE = true; - private static final String DEFAULT_REGION = null; private static final String INPUT_TYPE = "input"; private static final String OUTPUT_TYPE = "output"; private static final String DEFAULT_TYPE = "string"; @@ -83,17 +82,17 @@ public class TaskMapper { private static final Double EIB_TO_GIB = 1073741824.0; @Autowired - public TaskMapper(@Value("${cloud.pipeline.hddSize:}") Integer hddSize, - @Value("${cloud.pipeline.ramGb:}") Double defaultRamGb, - @Value("${cloud.pipeline.cpuCore:}") Long defaultCpuCore, - @Value("${cloud.pipeline.preemtible:}") Boolean defaultPreemptible, - @Value("${cloud.pipeline.region:}") String defaultRegion, + public TaskMapper(@Value("${cloud.pipeline.hddSize}") Integer hddSize, + @Value("${cloud.pipeline.ramGb}") Double defaultRamGb, + @Value("${cloud.pipeline.cpuCore}") Long defaultCpuCore, + @Value("${cloud.pipeline.preemtible}") Boolean defaultPreemptible, + @Value("${cloud.pipeline.region}") String defaultRegion, CloudPipelineAPIClient cloudPipelineAPIClient, MessageHelper messageHelper) { - this.defaultHddSize = hddSize != null ? hddSize : DEFAULT_HDD_SIZE; - this.defaultRamGb = defaultRamGb != null ? defaultRamGb : DEFAULT_RAM_GB; - this.defaultCpuCore = defaultCpuCore != null ? defaultCpuCore : DEFAULT_CPU_CORE; - this.defaultPreemptible = defaultPreemptible != null ? defaultPreemptible : DEFAULT_PREEMPRIBLE; - this.defaultRegion = StringUtils.isNoneEmpty(defaultRegion) ? defaultRegion : DEFAULT_REGION; + this.defaultHddSize = Optional.ofNullable(hddSize).orElse(DEFAULT_HDD_SIZE); + this.defaultRamGb = Optional.ofNullable(defaultRamGb).orElse(DEFAULT_RAM_GB); + this.defaultCpuCore = Optional.ofNullable(defaultCpuCore).orElse(DEFAULT_CPU_CORE); + this.defaultPreemptible = Optional.ofNullable(defaultPreemptible).orElse(DEFAULT_PREEMPRIBLE); + this.defaultRegion = defaultRegion; this.cloudPipelineAPIClient = cloudPipelineAPIClient; this.messageHelper = messageHelper; @@ -146,9 +145,10 @@ String getProperInstanceType(TesTask tesTask, Tool pipelineTool) { Long toolId = pipelineTool.getId(); Long regionId = getProperRegionIdInCloudRegionsByTesZone(Optional.ofNullable(tesTask.getResources()) .map(TesResources::getZones) - .orElseGet(() -> Collections.singletonList(Optional.ofNullable(defaultRegion).orElseThrow(() -> - new IllegalArgumentException(messageHelper - .getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, REGION)))))); + .orElseGet(() -> Collections.singletonList( + Optional.ofNullable(defaultRegion).filter(StringUtils::isNotEmpty).orElseThrow(() -> + new IllegalArgumentException(messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, REGION)))))); Boolean spot = Optional.ofNullable(tesTask.getResources()) .map(TesResources::getPreemptible).orElse(defaultPreemptible); AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = Optional.ofNullable(cloudPipelineAPIClient diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index fecc08a3e6..5d5343ca16 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -49,12 +49,12 @@ public class TesTaskServiceImpl implements TesTaskService { private static final Long DEFAULT_PAGE_SIZE = 256L; @Autowired - public TesTaskServiceImpl(@Value("${cloud.pipeline.service.name:}") String nameOfService, - @Value("${cloud.pipeline.doc:}") String doc, + public TesTaskServiceImpl(@Value("${cloud.pipeline.service.name}") String nameOfService, + @Value("${cloud.pipeline.doc}") String doc, CloudPipelineAPIClient cloudPipelineAPIClient, TaskMapper taskMapper, MessageHelper messageHelper) { - this.nameOfService = StringUtils.isNotEmpty(nameOfService) ? nameOfService : DEFAULT_NAME_SERVICE; - this.doc = StringUtils.isNotEmpty(doc) ? doc : DEFAULT_DOC; + this.nameOfService = Optional.ofNullable(nameOfService).filter(StringUtils::isNotEmpty).orElse(DEFAULT_NAME_SERVICE); + this.doc = Optional.ofNullable(doc).filter(StringUtils::isNotEmpty).orElse(DEFAULT_DOC); this.cloudPipelineAPIClient = cloudPipelineAPIClient; this.taskMapper = taskMapper; this.messageHelper = messageHelper; From 8ecd9afea814aaf21b37f5ecb57dc92d0e70612c Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Sat, 28 Sep 2019 16:39:45 +0300 Subject: [PATCH 154/194] replaced with Optional not-empty-null check for cloudPipelineHostUrl --- .../tesadapter/service/CloudPipelineAPIClient.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index cc64af12be..9fada94e56 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -18,12 +18,13 @@ import com.epam.pipeline.vo.PagingRunFilterExpressionVO; import com.epam.pipeline.vo.PagingRunFilterVO; import com.epam.pipeline.vo.RunStatusVO; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import org.springframework.util.Assert; import java.util.List; +import java.util.Optional; @Service public class CloudPipelineAPIClient { @@ -37,10 +38,10 @@ public class CloudPipelineAPIClient { public CloudPipelineAPIClient(TesTokenHolder tesTokenHolder, MessageHelper messageHelper, @Value("${cloud.pipeline.host}") String cloudPipelineHostUrl) { - Assert.hasText(cloudPipelineHostUrl, - messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, CLOUD_PIPELINE_HOST)); this.tesTokenHolder = tesTokenHolder; - this.cloudPipelineHostUrl = cloudPipelineHostUrl; + this.cloudPipelineHostUrl = Optional.ofNullable(cloudPipelineHostUrl).filter(StringUtils::isNotEmpty) + .orElseThrow(() -> new IllegalArgumentException(messageHelper.getMessage( + MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, CLOUD_PIPELINE_HOST))); } private CloudPipelineAPI buildCloudPipelineAPI() { From df97cc729d67a02f7004f96de0784c895953451c Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Mon, 30 Sep 2019 08:31:03 +0300 Subject: [PATCH 155/194] refactoring RestConfiguration --- .../configuration/RestConfiguration.java | 16 +++++++++------- .../controller/TesTokenInterceptor.java | 16 +++++++--------- .../test/resources/test-application.properties | 3 ++- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java index db96213f13..65910d5d64 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java @@ -1,7 +1,8 @@ package com.epam.pipeline.tesadapter.configuration; -import com.epam.pipeline.tesadapter.entity.TesTokenHolder; import com.epam.pipeline.tesadapter.controller.TesTokenInterceptor; +import com.epam.pipeline.tesadapter.entity.TesTokenHolder; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.http.HttpMessageConverters; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -17,6 +18,7 @@ @Configuration public class RestConfiguration implements WebMvcConfigurer { + private TesTokenInterceptor tesTokenInterceptor; @Bean public HttpMessageConverters customConverters() { @@ -26,12 +28,7 @@ public HttpMessageConverters customConverters() { @Override public void addInterceptors(InterceptorRegistry registry) { - registry.addInterceptor(getTesTokenInterceptor()); - } - - @Bean - public TesTokenInterceptor getTesTokenInterceptor() { - return new TesTokenInterceptor(getTesTokenHolder()); + registry.addInterceptor(tesTokenInterceptor); } @Bean @@ -39,4 +36,9 @@ public TesTokenInterceptor getTesTokenInterceptor() { public TesTokenHolder getTesTokenHolder() { return new TesTokenHolder(); } + + @Autowired + public void setTesTokenInterceptor(TesTokenInterceptor tesTokenInterceptor) { + this.tesTokenInterceptor = tesTokenInterceptor; + } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index ffa2634324..d597f58ab9 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -9,6 +9,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpHeaders; import org.springframework.security.web.util.matcher.IpAddressMatcher; +import org.springframework.stereotype.Component; import org.springframework.web.servlet.HandlerInterceptor; import javax.servlet.http.Cookie; @@ -19,6 +20,7 @@ @Slf4j @SuppressWarnings("unused") +@Component public class TesTokenInterceptor implements HandlerInterceptor { private static final String HTTP_AUTH_COOKIE = "HttpAuthorization"; @@ -27,12 +29,13 @@ public class TesTokenInterceptor implements HandlerInterceptor { @Value("${cloud.pipeline.token}") private String defaultPipelineToken; - @Value("${security.allowed.client.ip.range}") - private String ipRange; + private IpAddressMatcher ipAddressMatcher; @Autowired - public TesTokenInterceptor(TesTokenHolder tesTokenHolder) { + public TesTokenInterceptor(TesTokenHolder tesTokenHolder, + @Value("${security.allowed.client.ip.range}") String ipRange) { this.tesTokenHolder = tesTokenHolder; + ipAddressMatcher = new IpAddressMatcher(ipRange); } @Override @@ -60,11 +63,6 @@ private Optional checkRequestForToken(HttpServletRequest request) { } private boolean checkClientHostAddress(HttpServletRequest request) { - if (StringUtils.isNotEmpty(ipRange)) { - IpAddressMatcher ipAddressMatcher = new IpAddressMatcher(ipRange); - return ipAddressMatcher.matches(request); - } else { - return false; - } + return ipAddressMatcher != null && ipAddressMatcher.matches(request); } } diff --git a/tes-adapter/src/test/resources/test-application.properties b/tes-adapter/src/test/resources/test-application.properties index 9b7bd9e0e2..9ea90730a5 100644 --- a/tes-adapter/src/test/resources/test-application.properties +++ b/tes-adapter/src/test/resources/test-application.properties @@ -2,4 +2,5 @@ server.port=8080 cloud.pipeline.host=${API} cloud.pipeline.token=${API_TOKEN} cloud.pipeline.service.name=CloudPipeline -cloud.pipeline.doc=https://epam.github.io/cloud-pipeline/ \ No newline at end of file +cloud.pipeline.doc=https://epam.github.io/cloud-pipeline/ +security.allowed.client.ip.range=192.168.1.0/24 From 5a3a19631863ea100f205bad41c63b8ea42ffb03 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Mon, 30 Sep 2019 12:43:52 +0300 Subject: [PATCH 156/194] added bash script to get tes-adapter .jar file from git-repo --- .../scripts/run_get_tes-adapter_jar.sh | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 tes-adapter/scripts/run_get_tes-adapter_jar.sh diff --git a/tes-adapter/scripts/run_get_tes-adapter_jar.sh b/tes-adapter/scripts/run_get_tes-adapter_jar.sh new file mode 100644 index 0000000000..78c4c47e87 --- /dev/null +++ b/tes-adapter/scripts/run_get_tes-adapter_jar.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +workdir=$(pwd); + +if [ -n "$2" ]; then + gitRepoUrl="-b $1 $2" +else + gitRepoUrl=$1 +fi + +mkdir tmp; +cd tmp; + +tmpdir=$(pwd); + +git clone $gitRepoUrl $tmpdir; + +./gradlew bootJar; +echo "Copying jar-file from tes-adapter to $workdir" +cp -i tes-adapter/build/libs/tes-adapter-1.0-SNAPSHOT.jar ..; + +rm -r -f $tmpdir; +echo "tmp directory removed" \ No newline at end of file From 26bb5046504bc089b0963b6e99048d97c719792e Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Mon, 30 Sep 2019 14:29:25 +0300 Subject: [PATCH 157/194] refactoring RestConfiguration --- .../pipeline/tesadapter/controller/TesTokenInterceptor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index d597f58ab9..7455047724 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -29,13 +29,13 @@ public class TesTokenInterceptor implements HandlerInterceptor { @Value("${cloud.pipeline.token}") private String defaultPipelineToken; - private IpAddressMatcher ipAddressMatcher; + private final IpAddressMatcher ipAddressMatcher; @Autowired public TesTokenInterceptor(TesTokenHolder tesTokenHolder, @Value("${security.allowed.client.ip.range}") String ipRange) { this.tesTokenHolder = tesTokenHolder; - ipAddressMatcher = new IpAddressMatcher(ipRange); + ipAddressMatcher = StringUtils.isNotEmpty(ipRange) ? new IpAddressMatcher(ipRange) : null; } @Override From 5dad79156722313940d118049a5c7661ca17e71f Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Mon, 30 Sep 2019 17:46:32 +0300 Subject: [PATCH 158/194] updated bash script. Added exit-status check for necessary points --- .../scripts/run_get_tes-adapter_jar.sh | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tes-adapter/scripts/run_get_tes-adapter_jar.sh b/tes-adapter/scripts/run_get_tes-adapter_jar.sh index 78c4c47e87..51bab23737 100644 --- a/tes-adapter/scripts/run_get_tes-adapter_jar.sh +++ b/tes-adapter/scripts/run_get_tes-adapter_jar.sh @@ -10,14 +10,26 @@ fi mkdir tmp; cd tmp; - tmpdir=$(pwd); -git clone $gitRepoUrl $tmpdir; +#Step 1 +echo "Cloning git-repo from $gitRepoUrl" +git clone $gitRepoUrl $tmpdir || { echo "Cloning failed!" >&2; exit 1; }; + +#Step 2 +echo "Building jar-file..." +./gradlew bootJar &> /dev/null || { echo "Building jar-file failed!" >&2; exit 1; }; -./gradlew bootJar; +#Step 3 echo "Copying jar-file from tes-adapter to $workdir" -cp -i tes-adapter/build/libs/tes-adapter-1.0-SNAPSHOT.jar ..; +cp -i tes-adapter/build/libs/tes-adapter-1.0-SNAPSHOT.jar .. || { echo "Copying jar-file failed!" >&2; exit 1; }; + +#Check exit status and remove tmp files: +echo "Removing tmp directory..." +if [ $? -ne 0 ]; then + echo "The program failed! See errorlog file" +else + rm -r -f $tmpdir; echo "Success!" +fi + -rm -r -f $tmpdir; -echo "tmp directory removed" \ No newline at end of file From 5f04de17dab4319714c1233494f134b16d016e55 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Mon, 30 Sep 2019 17:52:48 +0300 Subject: [PATCH 159/194] updated bash script. changed printed echo-message if something going wrong --- tes-adapter/scripts/run_get_tes-adapter_jar.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tes-adapter/scripts/run_get_tes-adapter_jar.sh b/tes-adapter/scripts/run_get_tes-adapter_jar.sh index 51bab23737..681c618911 100644 --- a/tes-adapter/scripts/run_get_tes-adapter_jar.sh +++ b/tes-adapter/scripts/run_get_tes-adapter_jar.sh @@ -27,7 +27,7 @@ cp -i tes-adapter/build/libs/tes-adapter-1.0-SNAPSHOT.jar .. || { echo "Copying #Check exit status and remove tmp files: echo "Removing tmp directory..." if [ $? -ne 0 ]; then - echo "The program failed! See errorlog file" + echo "Something going wrong! Check log info!" else rm -r -f $tmpdir; echo "Success!" fi From f1b4aa14916fec6e6999a8e2d813b516791c612a Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 1 Oct 2019 19:02:32 +0300 Subject: [PATCH 160/194] refactored script, increased flexibility and stability --- tes-adapter/scripts/get_jar-file.sh | 52 +++++++++++++++++++ .../scripts/run_get_tes-adapter_jar.sh | 35 ------------- 2 files changed, 52 insertions(+), 35 deletions(-) create mode 100644 tes-adapter/scripts/get_jar-file.sh delete mode 100644 tes-adapter/scripts/run_get_tes-adapter_jar.sh diff --git a/tes-adapter/scripts/get_jar-file.sh b/tes-adapter/scripts/get_jar-file.sh new file mode 100644 index 0000000000..1a428a50e1 --- /dev/null +++ b/tes-adapter/scripts/get_jar-file.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +LOGFILE=script.log +# exit when any command fails +set -e +# keep track of the last executed command +trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG +# echo an error message before exiting +trap 'echo "\"${last_command}\" command filed with exit code $?." | tee -a $LOGFILE' ERR + +workdir=$(pwd) + +if [ -n "$2" ]; then + gitRepoUrl="-b $1 $2" +else + gitRepoUrl=$1 +fi + +echo "Creating tmp directory..." + +mkdir tmp +cd tmp +tmpdir=$(pwd) +TES_PATH=$tmpdir/tes-adapter +cd .. + +# Step 1. +# Cloning cloud-pipeline(CP) API from git repository, +# where $1 is URL(with possible specified branch(-b)), and +# $2 is where the CP will be copied. +echo "Cloning git-repo from $gitRepoUrl" +git clone $gitRepoUrl $tmpdir; + +# Step 2. +# Build jar file from tes-adapter module(TES-PATH) +echo "Building jar-file..." +cd tmp +if [ -d "$TES_PATH" ]; then + ./gradlew tes-adapter:bootJar > /dev/null 2>&1 +else + echo "The $TES_PATH directory doesn't exist!" 2>&1; exit 1 +fi + +# Step 3 +echo "Copying jar-file from tes-adapter to $workdir" +cp -rf tes-adapter/build/libs/tes-adapter-1.0-SNAPSHOT.jar .. + +# Remove tmp files: +echo "Removing tmp directory..." +rm -r -f $tmpdir + +echo "The Programm finished with Success!" diff --git a/tes-adapter/scripts/run_get_tes-adapter_jar.sh b/tes-adapter/scripts/run_get_tes-adapter_jar.sh deleted file mode 100644 index 681c618911..0000000000 --- a/tes-adapter/scripts/run_get_tes-adapter_jar.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -workdir=$(pwd); - -if [ -n "$2" ]; then - gitRepoUrl="-b $1 $2" -else - gitRepoUrl=$1 -fi - -mkdir tmp; -cd tmp; -tmpdir=$(pwd); - -#Step 1 -echo "Cloning git-repo from $gitRepoUrl" -git clone $gitRepoUrl $tmpdir || { echo "Cloning failed!" >&2; exit 1; }; - -#Step 2 -echo "Building jar-file..." -./gradlew bootJar &> /dev/null || { echo "Building jar-file failed!" >&2; exit 1; }; - -#Step 3 -echo "Copying jar-file from tes-adapter to $workdir" -cp -i tes-adapter/build/libs/tes-adapter-1.0-SNAPSHOT.jar .. || { echo "Copying jar-file failed!" >&2; exit 1; }; - -#Check exit status and remove tmp files: -echo "Removing tmp directory..." -if [ $? -ne 0 ]; then - echo "Something going wrong! Check log info!" -else - rm -r -f $tmpdir; echo "Success!" -fi - - From 3b0dc773e1f1b7e5d5a5e696cf9b6d60e8563d27 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 1 Oct 2019 22:26:42 +0300 Subject: [PATCH 161/194] couple clean-code in script --- tes-adapter/scripts/get_jar-file.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tes-adapter/scripts/get_jar-file.sh b/tes-adapter/scripts/get_jar-file.sh index 1a428a50e1..2be4e107bb 100644 --- a/tes-adapter/scripts/get_jar-file.sh +++ b/tes-adapter/scripts/get_jar-file.sh @@ -17,7 +17,6 @@ else fi echo "Creating tmp directory..." - mkdir tmp cd tmp tmpdir=$(pwd) @@ -34,8 +33,8 @@ git clone $gitRepoUrl $tmpdir; # Step 2. # Build jar file from tes-adapter module(TES-PATH) echo "Building jar-file..." -cd tmp if [ -d "$TES_PATH" ]; then + cd tmp ./gradlew tes-adapter:bootJar > /dev/null 2>&1 else echo "The $TES_PATH directory doesn't exist!" 2>&1; exit 1 @@ -49,4 +48,4 @@ cp -rf tes-adapter/build/libs/tes-adapter-1.0-SNAPSHOT.jar .. echo "Removing tmp directory..." rm -r -f $tmpdir -echo "The Programm finished with Success!" +echo "Success! You can find the corresponding jar-file in the folder $workdir" \ No newline at end of file From 415bfa4f44c90d51169904930f21d1ea982a5353 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 2 Oct 2019 13:52:44 +0300 Subject: [PATCH 162/194] refactored bash-script, added cleanup function, added trap cleanup for EXIT state, deleted Log-file --- tes-adapter/scripts/get_jar-file.sh | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tes-adapter/scripts/get_jar-file.sh b/tes-adapter/scripts/get_jar-file.sh index 2be4e107bb..6b01fbbc20 100644 --- a/tes-adapter/scripts/get_jar-file.sh +++ b/tes-adapter/scripts/get_jar-file.sh @@ -1,12 +1,20 @@ #!/bin/bash -LOGFILE=script.log # exit when any command fails set -e # keep track of the last executed command trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG # echo an error message before exiting -trap 'echo "\"${last_command}\" command filed with exit code $?." | tee -a $LOGFILE' ERR +trap 'echo "\"${last_command}\" command filed with exit code $?." cleanup' ERR + +# deletes the temp directory +function cleanup { + rm -rf $tmpdir + echo "Deleted temp directory $tmpdir" +} + +#cleanup before exiting +trap cleanup EXIT workdir=$(pwd) @@ -17,11 +25,10 @@ else fi echo "Creating tmp directory..." -mkdir tmp -cd tmp -tmpdir=$(pwd) + +tmpdir=$(mktemp -d -t ci-XXXXXXXXXX) TES_PATH=$tmpdir/tes-adapter -cd .. +TES_JAR_PATH=$TES_PATH/build/libs/tes-adapter-1.0-SNAPSHOT.jar # Step 1. # Cloning cloud-pipeline(CP) API from git repository, @@ -34,18 +41,14 @@ git clone $gitRepoUrl $tmpdir; # Build jar file from tes-adapter module(TES-PATH) echo "Building jar-file..." if [ -d "$TES_PATH" ]; then - cd tmp - ./gradlew tes-adapter:bootJar > /dev/null 2>&1 + cd $tmpdir + ./gradlew tes-adapter:bootJar >/dev/null 2>&1 else echo "The $TES_PATH directory doesn't exist!" 2>&1; exit 1 fi # Step 3 echo "Copying jar-file from tes-adapter to $workdir" -cp -rf tes-adapter/build/libs/tes-adapter-1.0-SNAPSHOT.jar .. - -# Remove tmp files: -echo "Removing tmp directory..." -rm -r -f $tmpdir +cp -rf $TES_JAR_PATH $workdir echo "Success! You can find the corresponding jar-file in the folder $workdir" \ No newline at end of file From 28c7745f0daca099eabf25a55210316c66199323 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 2 Oct 2019 13:56:01 +0300 Subject: [PATCH 163/194] refactored template of tempdir --- tes-adapter/scripts/get_jar-file.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tes-adapter/scripts/get_jar-file.sh b/tes-adapter/scripts/get_jar-file.sh index 6b01fbbc20..20667cd969 100644 --- a/tes-adapter/scripts/get_jar-file.sh +++ b/tes-adapter/scripts/get_jar-file.sh @@ -26,7 +26,7 @@ fi echo "Creating tmp directory..." -tmpdir=$(mktemp -d -t ci-XXXXXXXXXX) +tmpdir=$(mktemp -d -t cloud_pipeline-XXXXXXXXXX) TES_PATH=$tmpdir/tes-adapter TES_JAR_PATH=$TES_PATH/build/libs/tes-adapter-1.0-SNAPSHOT.jar From 4ccf58848819e62420e6dfa33eeffdb79efdeafb Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Mon, 30 Sep 2019 22:34:16 +0300 Subject: [PATCH 164/194] add Dockerfile --- tes-adapter/scripts/Dockerfile | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tes-adapter/scripts/Dockerfile diff --git a/tes-adapter/scripts/Dockerfile b/tes-adapter/scripts/Dockerfile new file mode 100644 index 0000000000..a5602abd1f --- /dev/null +++ b/tes-adapter/scripts/Dockerfile @@ -0,0 +1,13 @@ +# start from base +FROM centos:7 +# install +RUN apt-get -y install java-1.8.0-openjdk +# copy script +ADD scripFromEuv /init +# change rule and start script +RUN chmod +x /init +# expose port +EXPOSE 8080 +# start app +CMD ["/init"] + From cc99e0a816c80aa22004914a0aec4ea84408a4d3 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 1 Oct 2019 21:21:26 +0300 Subject: [PATCH 165/194] add Dockerfile --- tes-adapter/scripts/Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tes-adapter/scripts/Dockerfile b/tes-adapter/scripts/Dockerfile index a5602abd1f..27cc1a8ae4 100644 --- a/tes-adapter/scripts/Dockerfile +++ b/tes-adapter/scripts/Dockerfile @@ -3,11 +3,13 @@ FROM centos:7 # install RUN apt-get -y install java-1.8.0-openjdk # copy script -ADD scripFromEuv /init +ADD scriptFromEuvgenii /init # change rule and start script RUN chmod +x /init +# copy jar to /opt +COPY tes-adapter-1.0-SNAPSHOT.jar /opt +# run jar +CMD ["java", "-jar", "/opt/tes-adapter-1.0-SNAPSHOT.jar"] # expose port EXPOSE 8080 -# start app -CMD ["/init"] From d9514eab46fab7185b5d4a6a8a321039fe515522 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 1 Oct 2019 22:01:55 +0300 Subject: [PATCH 166/194] refactoring Dockerfile --- tes-adapter/scripts/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tes-adapter/scripts/Dockerfile b/tes-adapter/scripts/Dockerfile index 27cc1a8ae4..77da1f9d58 100644 --- a/tes-adapter/scripts/Dockerfile +++ b/tes-adapter/scripts/Dockerfile @@ -4,8 +4,10 @@ FROM centos:7 RUN apt-get -y install java-1.8.0-openjdk # copy script ADD scriptFromEuvgenii /init -# change rule and start script +# change rule RUN chmod +x /init +# start script +RUN ["/init/scriptFromEuvgenii.sh"] # copy jar to /opt COPY tes-adapter-1.0-SNAPSHOT.jar /opt # run jar From 0728c2c294d1d84a2855dd883caaa92e8307849b Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Thu, 3 Oct 2019 22:57:01 +0300 Subject: [PATCH 167/194] create Dockerfile --- tes-adapter/scripts/Dockerfile | 31 ++++++++++++++++++++--------- tes-adapter/scripts/get_jar-file.sh | 0 2 files changed, 22 insertions(+), 9 deletions(-) mode change 100644 => 100755 tes-adapter/scripts/get_jar-file.sh diff --git a/tes-adapter/scripts/Dockerfile b/tes-adapter/scripts/Dockerfile index 77da1f9d58..e6195feb4e 100644 --- a/tes-adapter/scripts/Dockerfile +++ b/tes-adapter/scripts/Dockerfile @@ -1,17 +1,30 @@ # start from base FROM centos:7 -# install -RUN apt-get -y install java-1.8.0-openjdk +# install tools +RUN yum install -y \ + git \ + wget \ + unzip +# url jdk 1.8.0 +ARG JAVA_SOURCE="https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u222-b10/OpenJDK8U-jdk_x64_linux_hotspot_8u222b10.tar.gz" +# download jdk and unzip it +RUN cd /usr/lib/ && \ + wget -P /usr/lib/ $JAVA_SOURCE && \ + tar -zxf OpenJDK8U-jdk_x64_linux_hotspot_8u222b10.tar.gz && \ + rm -f OpenJDK8U-jdk_x64_linux_hotspot_8u222b10.tar.gz +# set JAVA_HOME enviroment +ENV JAVA_HOME="/usr/lib/jdk8u222-b10/" +# set PATH enviroment +ENV PATH="$JAVA_HOME/bin:${PATH}" +ENV API="https://3.120.128.49:31080/pipeline/restapi/" +ENV API_TOKEN="eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJTVFVERU5UIiwidXNlcl9pZCI6IjExIiwicm9sZXMiOlsiUk9MRV9VU0VSIiwiUk9MRV9QSVBFTElORV9NQU5BR0VSIiwiUk9MRV9GT0xERVJfTUFOQUdFUiIsIlJPTEVfQ09ORklHVVJBVElPTl9NQU5BR0VSIl0sImdyb3VwcyI6W10sImV4cCI6MTU3Nzg5MTAwNiwiaWF0IjoxNTY3MTc3NDA3LCJqdGkiOiJkYTdjYTM1OC1mMDdjLTRkNmEtODE2ZS0zZWY5MTI0MjkzMTcifQ.DIbXGayBrYOgbeMKY0qP-TvvLRiG0cyFQnNJb6vKk8eenZQpu3P5Jb3NOi72z4oA_9NqI4RbjonPwSU0yycy0Alo5jcQxnyzqxWC1_5cA0Wo8Pi1MWnlYkrJr4H3Bs_FgUT87CGVSN8PuR2QAoBnzNfy7D1Qj8W4Hg95LULmhBE" # copy script -ADD scriptFromEuvgenii /init +ADD get_jar-file.sh /init # change rule RUN chmod +x /init # start script -RUN ["/init/scriptFromEuvgenii.sh"] -# copy jar to /opt -COPY tes-adapter-1.0-SNAPSHOT.jar /opt +RUN ["/init", "f_tes_application_properties_checking", "https://github.com/evgeniimv/cloud-pipeline.git"] # run jar -CMD ["java", "-jar", "/opt/tes-adapter-1.0-SNAPSHOT.jar"] +CMD ["java", "-jar", "tes-adapter-1.0-SNAPSHOT.jar"] # expose port -EXPOSE 8080 - +EXPOSE 8080 \ No newline at end of file diff --git a/tes-adapter/scripts/get_jar-file.sh b/tes-adapter/scripts/get_jar-file.sh old mode 100644 new mode 100755 From e96e43cce05d315c3da397f0bc2647a86da0a85b Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Fri, 4 Oct 2019 18:34:39 +0300 Subject: [PATCH 168/194] create Dockerfile --- tes-adapter/scripts/Dockerfile | 4 +--- tes-adapter/src/main/resources/application.properties | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tes-adapter/scripts/Dockerfile b/tes-adapter/scripts/Dockerfile index e6195feb4e..a73d743233 100644 --- a/tes-adapter/scripts/Dockerfile +++ b/tes-adapter/scripts/Dockerfile @@ -16,14 +16,12 @@ RUN cd /usr/lib/ && \ ENV JAVA_HOME="/usr/lib/jdk8u222-b10/" # set PATH enviroment ENV PATH="$JAVA_HOME/bin:${PATH}" -ENV API="https://3.120.128.49:31080/pipeline/restapi/" -ENV API_TOKEN="eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJTVFVERU5UIiwidXNlcl9pZCI6IjExIiwicm9sZXMiOlsiUk9MRV9VU0VSIiwiUk9MRV9QSVBFTElORV9NQU5BR0VSIiwiUk9MRV9GT0xERVJfTUFOQUdFUiIsIlJPTEVfQ09ORklHVVJBVElPTl9NQU5BR0VSIl0sImdyb3VwcyI6W10sImV4cCI6MTU3Nzg5MTAwNiwiaWF0IjoxNTY3MTc3NDA3LCJqdGkiOiJkYTdjYTM1OC1mMDdjLTRkNmEtODE2ZS0zZWY5MTI0MjkzMTcifQ.DIbXGayBrYOgbeMKY0qP-TvvLRiG0cyFQnNJb6vKk8eenZQpu3P5Jb3NOi72z4oA_9NqI4RbjonPwSU0yycy0Alo5jcQxnyzqxWC1_5cA0Wo8Pi1MWnlYkrJr4H3Bs_FgUT87CGVSN8PuR2QAoBnzNfy7D1Qj8W4Hg95LULmhBE" # copy script ADD get_jar-file.sh /init # change rule RUN chmod +x /init # start script -RUN ["/init", "f_tes_application_properties_checking", "https://github.com/evgeniimv/cloud-pipeline.git"] +RUN ["/init", "tes-support", "https://github.com/evgeniimv/cloud-pipeline.git"] # run jar CMD ["java", "-jar", "tes-adapter-1.0-SNAPSHOT.jar"] # expose port diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index 11d70349eb..3c8f6e5eb4 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -8,5 +8,5 @@ cloud.pipeline.preemtible=true cloud.pipeline.region=eu-central-1 cloud.pipeline.service.name=CloudPipeline cloud.pipeline.doc=https://epam.github.io/cloud-pipeline/ -security.allowed.client.ip.range=192.168.1.0/24 +security.allowed.client.ip.range=127.0.0.1/24 From fb59842f48ff1be90388cb1e1c8a27fa3b6b5de2 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Sun, 6 Oct 2019 16:28:56 +0300 Subject: [PATCH 169/194] refactoring Dockerfile --- tes-adapter/scripts/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tes-adapter/scripts/Dockerfile b/tes-adapter/scripts/Dockerfile index a73d743233..0ea7612ec6 100644 --- a/tes-adapter/scripts/Dockerfile +++ b/tes-adapter/scripts/Dockerfile @@ -21,7 +21,7 @@ ADD get_jar-file.sh /init # change rule RUN chmod +x /init # start script -RUN ["/init", "tes-support", "https://github.com/evgeniimv/cloud-pipeline.git"] +RUN ["/init", "tes-support", "https://github.com/SilinPavel/cloud-pipeline.git"] # run jar CMD ["java", "-jar", "tes-adapter-1.0-SNAPSHOT.jar"] # expose port From 04a2029119bfe8ef6e81cf7c377fade270166ade Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Fri, 4 Oct 2019 12:26:37 +0300 Subject: [PATCH 170/194] logging for TesTokenInterceptor --- .../tesadapter/common/MessageConstants.java | 5 +++++ .../controller/TesTokenInterceptor.java | 15 +++++++++++++-- .../src/main/resources/messages.properties | 6 +++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java index 9114df689e..0e20da7e06 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java @@ -9,4 +9,9 @@ public final class MessageConstants { //Parameters mapping public static final String ERROR_PARAMETER_NON_SCALAR_TYPE = "error.parameter.non.scalar.type"; + + //Auth messages + public static final String TOKEN_FOUND_IN_REQUEST = "debug.token.found.in.request"; + public static final String IP_ACCEPTED = "debug.ip.is.accepted"; + public static final String NO_MATCHED_AUTH_METHODS = "debug.no.matched.auth.methods"; } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index 7455047724..14ef680756 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -1,5 +1,7 @@ package com.epam.pipeline.tesadapter.controller; +import com.epam.pipeline.tesadapter.common.MessageConstants; +import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.entity.TesTokenHolder; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ArrayUtils; @@ -22,6 +24,7 @@ @SuppressWarnings("unused") @Component public class TesTokenInterceptor implements HandlerInterceptor { + private static final String HTTP_AUTH_COOKIE = "HttpAuthorization"; private TesTokenHolder tesTokenHolder; @@ -29,24 +32,32 @@ public class TesTokenInterceptor implements HandlerInterceptor { @Value("${cloud.pipeline.token}") private String defaultPipelineToken; + private MessageHelper messageHelper; + private final IpAddressMatcher ipAddressMatcher; @Autowired public TesTokenInterceptor(TesTokenHolder tesTokenHolder, + MessageHelper messageHelper, @Value("${security.allowed.client.ip.range}") String ipRange) { this.tesTokenHolder = tesTokenHolder; + this.messageHelper = messageHelper; ipAddressMatcher = StringUtils.isNotEmpty(ipRange) ? new IpAddressMatcher(ipRange) : null; } @Override public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { - if (checkRequestForToken(request).isPresent()) { - tesTokenHolder.setToken(checkRequestForToken(request).get()); + final Optional requestToken = checkRequestForToken(request); + if (requestToken.isPresent()) { + log.debug(messageHelper.getMessage(MessageConstants.TOKEN_FOUND_IN_REQUEST, request.getServletPath())); + tesTokenHolder.setToken(requestToken.get()); return true; } else if (checkClientHostAddress(request) && Strings.isNotEmpty(defaultPipelineToken)) { + log.debug(messageHelper.getMessage(MessageConstants.IP_ACCEPTED, request.getServletPath())); tesTokenHolder.setToken(defaultPipelineToken); return true; } + log.debug(messageHelper.getMessage(MessageConstants.NO_MATCHED_AUTH_METHODS, request.getServletPath())); response.sendError(HttpServletResponse.SC_UNAUTHORIZED); return false; } diff --git a/tes-adapter/src/main/resources/messages.properties b/tes-adapter/src/main/resources/messages.properties index 443bdc611b..75d2aa9503 100644 --- a/tes-adapter/src/main/resources/messages.properties +++ b/tes-adapter/src/main/resources/messages.properties @@ -5,4 +5,8 @@ error.parameter.required=Parameter ''{0}'' id required for ''{1}'' podId. #Parameters error.parameter.non.scalar.type=Unable to resolve parameter value ''{0}''. Reference value for field ''{1}'' is used as a parameter value. -error.parameter.incompatible.content=Parameter (id ''{0}'') with undesirable content. \ No newline at end of file +error.parameter.incompatible.content=Parameter (id ''{0}'') with undesirable content. + +debug.token.found.in.request=Auth token found in HEADERS or COOKIE for request ''{0}'' +debug.ip.is.accepted=Get default auth token, since request host is from accepted ip range for request ''{0}'' +debug.no.matched.auth.methods=No matched auth methods, send auth error for request ''{0}'' \ No newline at end of file From bfae3a52c3772524acf968a3b0f1a1176c44fb28 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Mon, 7 Oct 2019 13:06:35 +0300 Subject: [PATCH 171/194] added debug-logs for service, corresponded constants also added --- .../epam/pipeline/tesadapter/common/MessageConstants.java | 7 +++++++ .../pipeline/tesadapter/service/TesTaskServiceImpl.java | 6 ++++++ tes-adapter/src/main/resources/messages.properties | 8 +++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java index 0e20da7e06..4802c7d79c 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java @@ -14,4 +14,11 @@ public final class MessageConstants { public static final String TOKEN_FOUND_IN_REQUEST = "debug.token.found.in.request"; public static final String IP_ACCEPTED = "debug.ip.is.accepted"; public static final String NO_MATCHED_AUTH_METHODS = "debug.no.matched.auth.methods"; + public static final String PIPELINE_RUN_SUBMITTED = "debug.pipeline.run.submitted"; + public static final String GET_LIST_TASKS_BY_NAME_PREFIX = "debug.get.list.tasks.by.prefix"; + public static final String GET_LIST_TASKS_BY_DEFAULT_PREFIX = "debug.get.list.tasks.by.default"; + public static final String CANCEL_PIPELINE_RUN_BY_ID = "debug.cancel.pipeline.run.by.id"; + public static final String GET_PIPELINE_RUN_BY_ID = "debug.get.pipeline.run.by.id"; + public static final String GET_SERVICE_INFO = "debug.get.service.info"; + } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 5d5343ca16..522f99b3f0 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -66,6 +66,7 @@ public TesCreateTaskResponse submitTesTask(TesTask body) { PipelineRun pipelineRun = cloudPipelineAPIClient.runPipeline(taskMapper.mapToPipelineStart(body)); Assert.notNull(pipelineRun.getId(), messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_NULL_OR_EMPTY, ID)); + log.debug(messageHelper.getMessage(MessageConstants.PIPELINE_RUN_SUBMITTED, pipelineRun.getId())); tesCreateTaskResponse.setId(String.valueOf(pipelineRun.getId())); return tesCreateTaskResponse; } @@ -95,6 +96,7 @@ private List searchRunsWithNamePrefix(String namePrefix, Long pageS filterExpressionVO.setFilterExpression(expression); filterExpressionVO.setPage(Integer.parseInt(Optional.ofNullable(pageToken).orElse(DEFAULT_PAGE_TOKEN))); filterExpressionVO.setPageSize(Optional.ofNullable(pageSize).orElse(DEFAULT_PAGE_SIZE).intValue()); + log.debug(messageHelper.getMessage(MessageConstants.GET_LIST_TASKS_BY_NAME_PREFIX, namePrefix)); return ListUtils.emptyIfNull(cloudPipelineAPIClient.searchRuns(filterExpressionVO).getElements()); } @@ -102,6 +104,7 @@ private List filterRunsWithOutNamePrefix(Long pageSize, String page PagingRunFilterVO filterVO = new PagingRunFilterVO(); filterVO.setPage(Integer.parseInt(Optional.ofNullable(pageToken).orElse(DEFAULT_PAGE_TOKEN))); filterVO.setPageSize(Optional.ofNullable(pageSize).orElse(DEFAULT_PAGE_SIZE).intValue()); + log.debug(messageHelper.getMessage(MessageConstants.GET_LIST_TASKS_BY_DEFAULT_PREFIX)); return ListUtils.emptyIfNull(cloudPipelineAPIClient.filterRuns(filterVO, LOAD_STORAGE_LINKS).getElements()); } @@ -110,11 +113,13 @@ public TesCancelTaskResponse cancelTesTask(String id) { RunStatusVO updateStatus = new RunStatusVO(); updateStatus.setStatus(TaskStatus.STOPPED); cloudPipelineAPIClient.updateRunStatus(parseRunId(id), updateStatus); + log.debug(messageHelper.getMessage(MessageConstants.CANCEL_PIPELINE_RUN_BY_ID, id)); return new TesCancelTaskResponse(); } @Override public TesTask getTesTask(String id, TaskView view) { + log.debug(messageHelper.getMessage(MessageConstants.GET_PIPELINE_RUN_BY_ID, id)); return taskMapper.mapToTesTask(cloudPipelineAPIClient.loadPipelineRun(parseRunId(id)), view); } @@ -126,6 +131,7 @@ private Long parseRunId(String id) { @Override public TesServiceInfo getServiceInfo() { + log.debug(messageHelper.getMessage(MessageConstants.GET_SERVICE_INFO)); final TesServiceInfo tesServiceInfo = new TesServiceInfo(); tesServiceInfo.setName(nameOfService); tesServiceInfo.setDoc(doc); diff --git a/tes-adapter/src/main/resources/messages.properties b/tes-adapter/src/main/resources/messages.properties index 75d2aa9503..113bbc914d 100644 --- a/tes-adapter/src/main/resources/messages.properties +++ b/tes-adapter/src/main/resources/messages.properties @@ -9,4 +9,10 @@ error.parameter.incompatible.content=Parameter (id ''{0}'') with undesirable con debug.token.found.in.request=Auth token found in HEADERS or COOKIE for request ''{0}'' debug.ip.is.accepted=Get default auth token, since request host is from accepted ip range for request ''{0}'' -debug.no.matched.auth.methods=No matched auth methods, send auth error for request ''{0}'' \ No newline at end of file +debug.no.matched.auth.methods=No matched auth methods, send auth error for request ''{0}'' +debug.pipeline.run.submitted=Pipeline-run from request tes-body is submitted, corresponding pipeline-run id is ''{0}'' +debug.get.list.tasks.by.prefix=Get a list of tasks by the corresponding name-prefix = ''{0}'' +debug.get.list.tasks.by.default=Get a list of tasks by the default name-prefix +debug.cancel.pipeline.run.by.id=Corresponded pipeline-run with id = ''{0}'' is canceled +debug.get.pipeline.run.by.id=Try ro get pipeline-run with id = ''{0}'' +debug.get.service.info=Try to get service info \ No newline at end of file From 71e2f7996a27a813d28cfda91150a33d34fc558b Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Mon, 7 Oct 2019 15:45:00 +0300 Subject: [PATCH 172/194] add support ipv6 --- .../tesadapter/controller/TesTokenInterceptor.java | 12 ++++++++---- .../src/main/resources/application.properties | 4 +++- .../src/test/resources/test-application.properties | 3 ++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index 14ef680756..3f10e5d741 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -34,15 +34,18 @@ public class TesTokenInterceptor implements HandlerInterceptor { private MessageHelper messageHelper; - private final IpAddressMatcher ipAddressMatcher; + private final IpAddressMatcher ipAddressMatcherV4; + private final IpAddressMatcher ipAddressMatcherV6; @Autowired public TesTokenInterceptor(TesTokenHolder tesTokenHolder, MessageHelper messageHelper, - @Value("${security.allowed.client.ip.range}") String ipRange) { + @Value("${security.allowed.client.ipv4.range}") String ipRangeV4, + @Value("${security.allowed.client.ipv6.range}") String ipRangeV6) { this.tesTokenHolder = tesTokenHolder; this.messageHelper = messageHelper; - ipAddressMatcher = StringUtils.isNotEmpty(ipRange) ? new IpAddressMatcher(ipRange) : null; + ipAddressMatcherV4 = StringUtils.isNotEmpty(ipRangeV4) ? new IpAddressMatcher(ipRangeV4) : null; + ipAddressMatcherV6 = StringUtils.isNotEmpty(ipRangeV6) ? new IpAddressMatcher(ipRangeV6) : null; } @Override @@ -74,6 +77,7 @@ private Optional checkRequestForToken(HttpServletRequest request) { } private boolean checkClientHostAddress(HttpServletRequest request) { - return ipAddressMatcher != null && ipAddressMatcher.matches(request); + return (ipAddressMatcherV4 != null && ipAddressMatcherV4.matches(request)) || + (ipAddressMatcherV6 != null && ipAddressMatcherV6.matches(request)); } } diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index 3c8f6e5eb4..649f97cbec 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -8,5 +8,7 @@ cloud.pipeline.preemtible=true cloud.pipeline.region=eu-central-1 cloud.pipeline.service.name=CloudPipeline cloud.pipeline.doc=https://epam.github.io/cloud-pipeline/ -security.allowed.client.ip.range=127.0.0.1/24 +security.allowed.client.ipv4.range=127.0.0.1/24 +security.allowed.client.ipv6.range=::1/128 + diff --git a/tes-adapter/src/test/resources/test-application.properties b/tes-adapter/src/test/resources/test-application.properties index 9ea90730a5..edea56b816 100644 --- a/tes-adapter/src/test/resources/test-application.properties +++ b/tes-adapter/src/test/resources/test-application.properties @@ -3,4 +3,5 @@ cloud.pipeline.host=${API} cloud.pipeline.token=${API_TOKEN} cloud.pipeline.service.name=CloudPipeline cloud.pipeline.doc=https://epam.github.io/cloud-pipeline/ -security.allowed.client.ip.range=192.168.1.0/24 +security.allowed.client.ipv4.range=192.168.1.0/24 +security.allowed.client.ipv6.range=::1/128 From 6c47a1f36a91449b437be77ae8ab9273ffb8805b Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Mon, 7 Oct 2019 13:11:24 +0300 Subject: [PATCH 173/194] added deploy relations for tes-adapter, added stubbed files for k&s and Docker, and modifications in corresponded *.sh files --- deploy/contents/install/app/install.sh | 26 ++++++++++++++++--- .../k8s/cp-tes-srv/cp-tes-srv-dpl.yaml | 1 + .../k8s/cp-tes-srv/cp-tes-srv-svc.yaml | 0 deploy/docker/build-dockers.sh | 6 +++++ deploy/docker/cp-tes-srv/Dockerfile | 15 +++++++++++ .../cp-tes-srv/config/application.properties | 0 6 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml create mode 100644 deploy/contents/k8s/cp-tes-srv/cp-tes-srv-svc.yaml create mode 100644 deploy/docker/cp-tes-srv/Dockerfile create mode 100644 deploy/docker/cp-tes-srv/config/application.properties diff --git a/deploy/contents/install/app/install.sh b/deploy/contents/install/app/install.sh index 787e86a9b6..5bb41de0e0 100644 --- a/deploy/contents/install/app/install.sh +++ b/deploy/contents/install/app/install.sh @@ -497,7 +497,7 @@ if is_service_requested cp-api-srv; then print_info "-> Registering custom users in IdP and API services" api_register_custom_users "$CP_CUSTOM_USERS_SPEC" - # Here we wait for the price list sync, this is required by the downstream services to manage the instance types configurations + # Here we wait for the price list sync, this is required by the downstream services to manage the instance types configurations if [ -z "$CP_CLOUD_REGION_INTERNAL_ID" ]; then print_warn "CP_CLOUD_REGION_INTERNAL_ID is not defined, assuming that a cloud region is not registered correctly previously. WILL NOT wait for price lists synchonization" else @@ -595,7 +595,7 @@ if is_service_requested cp-edge; then delete_deployment_and_service "cp-edge" \ "/opt/edge" - if is_install_requested; then + if is_install_requested; then print_info "-> Creating self-signed SSL certificate for EDGE (${CP_EDGE_EXTERNAL_HOST}, ${CP_EDGE_INTERNAL_HOST})" generate_self_signed_key_pair $CP_EDGE_CERT_DIR/ssl-private-key.pem \ $CP_EDGE_CERT_DIR/ssl-public-cert.pem \ @@ -616,7 +616,7 @@ if is_service_requested cp-edge; then update_config_value "$CP_INSTALL_CONFIG_FILE" \ "EDGE_EXTERNAL" \ "$EDGE_EXTERNAL" - + init_kube_config_map print_ok "-> EDGE addresses parameters set:" @@ -905,7 +905,7 @@ if is_service_requested cp-search; then print_info "-> Deleting existing instance of Search ELK service" delete_deployment_and_service "cp-search-elk" \ - "/opt/search-elk" + "/opt/search-elk" if is_install_requested; then print_info "-> Deploying Search ELK service" @@ -1014,6 +1014,24 @@ if is_service_requested cp-share-srv; then echo fi +# TES Service +if is_service_requested cp-tes-srv; then +print_ok "[Starting TES Service deployment]" + + print_info "-> Deleting existing instance of TES Service" + delete_deployment_and_service "cp-tes-srv" \ + "/opt/tes-srv" + if is_install_requested; then + print_info "-> Deploying TES service" + create_kube_resource $K8S_SPECS_HOME/cp-tes-srv/cp-tes-srv-dpl.yaml + + print_info "-> Waiting for TES service to initialize" + wait_for_deployment "cp-tes-srv" + + CP_INSTALL_SUMMARY="$CP_INSTALL_SUMMARY\ncp-tes-srv: deployed" + fi + echo +fi print_ok "Installation done" echo -e $CP_INSTALL_SUMMARY diff --git a/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml new file mode 100644 index 0000000000..5678aa267e --- /dev/null +++ b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml @@ -0,0 +1 @@ +# FIXME Add spec implementation \ No newline at end of file diff --git a/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-svc.yaml b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-svc.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deploy/docker/build-dockers.sh b/deploy/docker/build-dockers.sh index 45c382f6d2..e96c84f7a1 100644 --- a/deploy/docker/build-dockers.sh +++ b/deploy/docker/build-dockers.sh @@ -200,6 +200,12 @@ docker build $DOCKERS_SOURCES_PATH/cp-share-srv \ --build-arg CP_API_DIST_URL="$CP_API_DIST_URL" docker push "$CP_SHARE_SRV_DIST_NAME" +# TES Service +CP_TES_SRV_DIST_NAME=${CP_TES_SRV_DIST_NAME:-"$CP_DIST_REPO_NAME:tes-srv-${DOCKERS_VERSION}"} +docker build $DOCKERS_SOURCES_PATH/cp-tes-srv \ + -t "$CP_TES_SRV_DIST_NAME" +docker push "$CP_TES_SRV_DIST_NAME" + ######################## # Base tools dockers ######################## diff --git a/deploy/docker/cp-tes-srv/Dockerfile b/deploy/docker/cp-tes-srv/Dockerfile new file mode 100644 index 0000000000..e545f889c1 --- /dev/null +++ b/deploy/docker/cp-tes-srv/Dockerfile @@ -0,0 +1,15 @@ +# Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# FIXME Docker implementation should be here diff --git a/deploy/docker/cp-tes-srv/config/application.properties b/deploy/docker/cp-tes-srv/config/application.properties new file mode 100644 index 0000000000..e69de29bb2 From 989df8ddb70d5b6ffdefc9d31053dab2ab98486e Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 8 Oct 2019 13:12:24 +0300 Subject: [PATCH 174/194] Did a few clean-code changes for tes-adapter module, checkstyle and PMD checks for whole sub-module. --- tes-adapter/scripts/Dockerfile | 7 +++--- .../tesadapter/common/MessageConstants.java | 1 + .../controller/TesAdapterController.java | 6 ++--- .../controller/TesTokenInterceptor.java | 3 ++- .../app/TesAdapterControllerTest.java | 23 ++++++++++--------- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/tes-adapter/scripts/Dockerfile b/tes-adapter/scripts/Dockerfile index 0ea7612ec6..ebd31510aa 100644 --- a/tes-adapter/scripts/Dockerfile +++ b/tes-adapter/scripts/Dockerfile @@ -7,9 +7,10 @@ RUN yum install -y \ unzip # url jdk 1.8.0 ARG JAVA_SOURCE="https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u222-b10/OpenJDK8U-jdk_x64_linux_hotspot_8u222b10.tar.gz" +# set /usr/lib as WORKDIR +WORKDIR /usr/lib # download jdk and unzip it -RUN cd /usr/lib/ && \ - wget -P /usr/lib/ $JAVA_SOURCE && \ +RUN wget -P /usr/lib/ $JAVA_SOURCE && \ tar -zxf OpenJDK8U-jdk_x64_linux_hotspot_8u222b10.tar.gz && \ rm -f OpenJDK8U-jdk_x64_linux_hotspot_8u222b10.tar.gz # set JAVA_HOME enviroment @@ -17,7 +18,7 @@ ENV JAVA_HOME="/usr/lib/jdk8u222-b10/" # set PATH enviroment ENV PATH="$JAVA_HOME/bin:${PATH}" # copy script -ADD get_jar-file.sh /init +COPY get_jar-file.sh /init # change rule RUN chmod +x /init # start script diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java index 4802c7d79c..6991d8fa7b 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java @@ -1,5 +1,6 @@ package com.epam.pipeline.tesadapter.common; +@SuppressWarnings({"PMD.UseUtilityClass", "checkstyle:HideUtilityClassConstructor"}) public final class MessageConstants { //Common errors diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java index 0e53bc9c59..2df70da66c 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesAdapterController.java @@ -44,18 +44,18 @@ public ResponseEntity listTesTasks( } @PostMapping("/v1/tasks") - ResponseEntity submitTesTask(@RequestBody TesTask body) { + public ResponseEntity submitTesTask(@RequestBody TesTask body) { return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.submitTesTask(body)); } @GetMapping("/v1/tasks/{id}") - ResponseEntity getTesTask(@PathVariable String id, @RequestParam(required = false, + public ResponseEntity getTesTask(@PathVariable String id, @RequestParam(required = false, defaultValue = "MINIMAL") TaskView view) { return ResponseEntity.ok().body(tesTaskService.getTesTask(id, view)); } @PostMapping("/v1/tasks/{id}:cancel") - ResponseEntity cancelTesTask(@PathVariable String id) { + public ResponseEntity cancelTesTask(@PathVariable String id) { return ResponseEntity.status(HttpStatus.OK).body(tesTaskService.cancelTesTask(id)); } } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index 3f10e5d741..f5ec06046e 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -49,7 +49,8 @@ public TesTokenInterceptor(TesTokenHolder tesTokenHolder, } @Override - public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) + throws Exception { final Optional requestToken = checkRequestForToken(request); if (requestToken.isPresent()) { log.debug(messageHelper.getMessage(MessageConstants.TOKEN_FOUND_IN_REQUEST, request.getServletPath())); diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 1b6c75b816..9d2c9d526a 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -40,7 +40,8 @@ @WebMvcTest(TesAdapterController.class) -@SuppressWarnings({"unused", "PMD.TooManyStaticImports", "PMD.AvoidUsingHardCodedIP"}) +@SuppressWarnings({"unused", "PMD.TooManyStaticImports", "PMD.AvoidUsingHardCodedIP", + "PMD.JUnitTestsShouldIncludeAssert"}) @TestPropertySource(locations = "classpath:test-application.properties") public class TesAdapterControllerTest { private static final String DEFAULT_TASK_ID = "5"; @@ -89,7 +90,7 @@ public class TesAdapterControllerTest { private TesExecutor tesExecutor = new TesExecutor(); @BeforeEach - void setUp() { + private void setUp() { when(tesTaskService.cancelTesTask(DEFAULT_TASK_ID)).thenReturn(new TesCancelTaskResponse()); when(tesTaskService.listTesTask(NAME_PREFIX, PAGE_SIZE, PAGE_TOKEN, DEFAULT_VIEW)) @@ -102,7 +103,7 @@ void setUp() { } @Test - void submitTesTaskWhenRequestingTesTaskBodyAndReturnId() throws Exception { + public void submitTesTaskWhenRequestingTesTaskBodyAndReturnId() throws Exception { tesCreateTaskResponse.setId(DEFAULT_TASK_ID); when(tesTaskService.submitTesTask(any(TesTask.class))).thenReturn(tesCreateTaskResponse); this.mockMvc.perform(post("/v1/tasks") @@ -114,7 +115,7 @@ void submitTesTaskWhenRequestingTesTaskBodyAndReturnId() throws Exception { } @Test - void expectIllegalArgExceptionWhenRunSubmitTesTasWithNullId() throws Exception { + public void expectIllegalArgExceptionWhenRunSubmitTesTasWithNullId() throws Exception { tesTask.setExecutors(null); when(tesTaskService.submitTesTask(tesTask)).thenThrow(new IllegalArgumentException()); this.mockMvc.perform(post("/v1/tasks") @@ -125,7 +126,7 @@ void expectIllegalArgExceptionWhenRunSubmitTesTasWithNullId() throws Exception { } @Test - void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { + public void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { when(tesTaskService.cancelTesTask(DEFAULT_TASK_ID)).thenReturn(new TesCancelTaskResponse()); this.mockMvc.perform(post("/v1/tasks/{id}:cancel", DEFAULT_TASK_ID) .header(HttpHeaders.AUTHORIZATION, defaultPipelineToken)) @@ -133,7 +134,7 @@ void cancelTesTaskWhenRequestingIdReturnCanceledTask() throws Exception { } @Test - void expectIllegalStateExceptionWhenRunCancelTesTaskWithWrongId() throws Exception { + public void expectIllegalStateExceptionWhenRunCancelTesTaskWithWrongId() throws Exception { when(tesTaskService.cancelTesTask(EMPTY_INPUT)).thenThrow(new IllegalStateException(messageHelper .getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, "taskId"))); this.mockMvc.perform(post("/v1/tasks/{id}:cancel", EMPTY_INPUT) @@ -146,7 +147,7 @@ void expectIllegalStateExceptionWhenRunCancelTesTaskWithWrongId() throws Excepti } @Test - void listTesTaskWhenRequestingReturnTesListTasksResponse() throws Exception { + public void listTesTaskWhenRequestingReturnTesListTasksResponse() throws Exception { this.mockMvc.perform(get("/v1/tasks?name_prefix={name_prefix}?page_size={page_size}" + "?page_token={page_token}?view={view}", NAME_PREFIX, PAGE_SIZE, PAGE_TOKEN, DEFAULT_VIEW) @@ -155,7 +156,7 @@ void listTesTaskWhenRequestingReturnTesListTasksResponse() throws Exception { } @Test - void getTesTaskWhenRequestingReturnTesTaskResponse() throws Exception { + public void getTesTaskWhenRequestingReturnTesTaskResponse() throws Exception { tesExecutor.setImage(DEFAULT_IMAGE); tesExecutor.setCommand(Collections.singletonList(DEFAULT_COMMAND)); tesTask.setExecutors(Collections.singletonList(tesExecutor)); @@ -167,7 +168,7 @@ void getTesTaskWhenRequestingReturnTesTaskResponse() throws Exception { } @Test - void serviceInfoRequestShouldReturnCurrentServiceState() throws Exception { + public void serviceInfoRequestShouldReturnCurrentServiceState() throws Exception { this.mockMvc.perform(get("/v1/tasks/service-info") .header(HttpHeaders.AUTHORIZATION, defaultPipelineToken) .contentType(JSON_CONTENT_TYPE)) @@ -176,7 +177,7 @@ void serviceInfoRequestShouldReturnCurrentServiceState() throws Exception { } @Test - void preHandleMethodShouldCheckAuthorizationContext() throws Exception { + public void preHandleMethodShouldCheckAuthorizationContext() throws Exception { this.mockMvc.perform(get(GET_SERVICE_INFO) .header(HttpHeaders.AUTHORIZATION, defaultPipelineToken).contentType(JSON_CONTENT_TYPE)) .andDo(print()).andExpect(status().isOk()) @@ -193,7 +194,7 @@ void preHandleMethodShouldCheckAuthorizationContext() throws Exception { } @Test - void preHandleMethodShouldCheckAuthorizationContextWithIpRange() throws Exception { + public void preHandleMethodShouldCheckAuthorizationContextWithIpRange() throws Exception { this.mockMvc.perform(get(GET_SERVICE_INFO) .with(request -> { request.setRemoteAddr(IP_IN_RANGE); From 843f2a3236be5c65ced7c25ffc0d9306f0c474d6 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Mon, 7 Oct 2019 19:40:27 +0300 Subject: [PATCH 175/194] add dpl.yaml --- .../k8s/cp-tes-srv/cp-tes-srv-dpl.yaml | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml index 5678aa267e..9a5c2e1129 100644 --- a/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml +++ b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml @@ -1 +1,25 @@ -# FIXME Add spec implementation \ No newline at end of file +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: cp-tes-srv + namespace: default +spec: + replicas: 1 + template: + metadata: + name: cp-git-sync + namespace: default + labels: + cloud-pipeline/cp-tes-srv: "true" + spec: + nodeSelector: + cloud-pipeline/cp-tes-srv: "true" + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + - name: cp-tes-srv + image: lifescience/cloud-pipeline:tes-srv-$CP_VERSION + imagePullPolicy: "Always" + ports: + - containerPort: 8080 From 48610a30aa8126639d31d1636c33b7e071ce41ac Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 8 Oct 2019 17:18:53 +0300 Subject: [PATCH 176/194] refactoring cp-tes-erv-dpl.yaml --- deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml index 9a5c2e1129..40e70fe508 100644 --- a/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml +++ b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml @@ -7,7 +7,7 @@ spec: replicas: 1 template: metadata: - name: cp-git-sync + name: cp-tes-srv namespace: default labels: cloud-pipeline/cp-tes-srv: "true" @@ -20,6 +20,6 @@ spec: containers: - name: cp-tes-srv image: lifescience/cloud-pipeline:tes-srv-$CP_VERSION - imagePullPolicy: "Always" + imagePullPolicy: "ifNotPresent" ports: - containerPort: 8080 From ab44535bf7515bf8a1c8130f5da313b7cbb4eac4 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 8 Oct 2019 21:15:39 +0300 Subject: [PATCH 177/194] refactoring cp-tes-erv-dpl.yaml --- deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml | 3 +++ tes-adapter/scripts/Dockerfile | 2 ++ tes-adapter/scripts/application.properties | 4 ++++ 3 files changed, 9 insertions(+) create mode 100644 tes-adapter/scripts/application.properties diff --git a/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml index 40e70fe508..93e4356a8c 100644 --- a/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml +++ b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml @@ -23,3 +23,6 @@ spec: imagePullPolicy: "ifNotPresent" ports: - containerPort: 8080 + envFrom: + - configMapRef: + name: cp-config-global diff --git a/tes-adapter/scripts/Dockerfile b/tes-adapter/scripts/Dockerfile index ebd31510aa..f4401386b2 100644 --- a/tes-adapter/scripts/Dockerfile +++ b/tes-adapter/scripts/Dockerfile @@ -23,6 +23,8 @@ COPY get_jar-file.sh /init RUN chmod +x /init # start script RUN ["/init", "tes-support", "https://github.com/SilinPavel/cloud-pipeline.git"] +# copy application.properties +COPY application.properties / # run jar CMD ["java", "-jar", "tes-adapter-1.0-SNAPSHOT.jar"] # expose port diff --git a/tes-adapter/scripts/application.properties b/tes-adapter/scripts/application.properties new file mode 100644 index 0000000000..bcd3b0fbc3 --- /dev/null +++ b/tes-adapter/scripts/application.properties @@ -0,0 +1,4 @@ +cloud.pipeline.host=https://${CP_API_SRV_INTERNAL_HOST:cp-api-srv.default.svc.cluster.local}:${CP_API_SRV_INTERNAL_PORT:31080}/pipeline/ +cloud.pipeline.token=${CP_API_JWT_ADMIN} + + From e886cc4b357616d21aa75bfc40ed8e1504d488dc Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 8 Oct 2019 22:35:08 +0300 Subject: [PATCH 178/194] Added service specs for k&s deployment, also added couple corresponded modifications to install.sh script --- deploy/contents/install/app/install.sh | 10 ++++++++++ .../cp-tes-srv/cp-tes-srv-svc-external-ip.yaml | 17 +++++++++++++++++ .../cp-tes-srv/cp-tes-srv-svc-node-port.yaml | 17 +++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 deploy/contents/k8s/cp-tes-srv/cp-tes-srv-svc-external-ip.yaml create mode 100644 deploy/contents/k8s/cp-tes-srv/cp-tes-srv-svc-node-port.yaml diff --git a/deploy/contents/install/app/install.sh b/deploy/contents/install/app/install.sh index 5bb41de0e0..3552de6924 100644 --- a/deploy/contents/install/app/install.sh +++ b/deploy/contents/install/app/install.sh @@ -238,6 +238,11 @@ CP_VM_MONITOR_KUBE_NODE_NAME=${CP_VM_MONITOR_KUBE_NODE_NAME:-$KUBE_MASTER_NODE_N print_info "-> Assigning cloud-pipeline/cp-share-srv to $CP_VM_MONITOR_KUBE_NODE_NAME" kubectl label nodes "$CP_VM_MONITOR_KUBE_NODE_NAME" cloud-pipeline/cp-share-srv="true" --overwrite +# Allow to schedule TES service to the master +CP_TES_SRV_KUBE_NODE_NAME=${CP_TES_SRV_KUBE_NODE_NAME:-$KUBE_MASTER_NODE_NAME} +print_info "-> Assigning cloud-pipeline/cp-tes-srv to $CP_TES_SRV_KUBE_NODE_NAME" +kubectl label nodes "$CP_TES_SRV_KUBE_NODE_NAME" cloud-pipeline/cp-tes-srv="true" --overwrite + echo ########## @@ -1024,10 +1029,15 @@ print_ok "[Starting TES Service deployment]" if is_install_requested; then print_info "-> Deploying TES service" create_kube_resource $K8S_SPECS_HOME/cp-tes-srv/cp-tes-srv-dpl.yaml + create_kube_resource $K8S_SPECS_HOME/cp-tes-srv/cp-tes-srv-svc.yaml --svc print_info "-> Waiting for TES service to initialize" wait_for_deployment "cp-tes-srv" + expose_cluster_port "cp-tes-srv" \ + "31086" \ + "8080" + CP_INSTALL_SUMMARY="$CP_INSTALL_SUMMARY\ncp-tes-srv: deployed" fi echo diff --git a/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-svc-external-ip.yaml b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-svc-external-ip.yaml new file mode 100644 index 0000000000..4566a612ef --- /dev/null +++ b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-svc-external-ip.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + cloud-pipeline/cp-tes-srv: "true" + name: cp-tes-srv + namespace: default +spec: + ports: + - protocol: TCP + port: 32082 + targetPort: 8080 + name: cp-tes-srv-port-https + externalIPs: +${CP_TES_SRV_SVC_EXTERNAL_IP_LIST} + selector: + cloud-pipeline/cp-tes-srv: "true" diff --git a/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-svc-node-port.yaml b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-svc-node-port.yaml new file mode 100644 index 0000000000..06449ac6cb --- /dev/null +++ b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-svc-node-port.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + cloud-pipeline/cp-tes-srv: "true" + name: cp-tes-srv + namespace: default +spec: + ports: + - protocol: TCP + nodePort: 32082 + port: 32082 + targetPort: 8080 + name: cp-tes-srv-port-https + type: NodePort + selector: + cloud-pipeline/cp-tes-srv: "true" From bc8667322e36554283ade244c117fdf44691e4a6 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 8 Oct 2019 22:38:53 +0300 Subject: [PATCH 179/194] removed expose_cluster_port implementation in install.sh script for TES --- deploy/contents/install/app/install.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/deploy/contents/install/app/install.sh b/deploy/contents/install/app/install.sh index 3552de6924..91d55e5c37 100644 --- a/deploy/contents/install/app/install.sh +++ b/deploy/contents/install/app/install.sh @@ -1034,10 +1034,6 @@ print_ok "[Starting TES Service deployment]" print_info "-> Waiting for TES service to initialize" wait_for_deployment "cp-tes-srv" - expose_cluster_port "cp-tes-srv" \ - "31086" \ - "8080" - CP_INSTALL_SUMMARY="$CP_INSTALL_SUMMARY\ncp-tes-srv: deployed" fi echo From ba4ca172ffbf16d358eee281760442c4bed474de Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Wed, 9 Oct 2019 12:39:22 +0300 Subject: [PATCH 180/194] delete unused kube config file --- deploy/contents/k8s/cp-tes-srv/cp-tes-srv-svc.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 deploy/contents/k8s/cp-tes-srv/cp-tes-srv-svc.yaml diff --git a/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-svc.yaml b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-svc.yaml deleted file mode 100644 index e69de29bb2..0000000000 From 98317fd4dbf6276195cff2b9d525f084c106f0b7 Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Wed, 9 Oct 2019 12:50:21 +0300 Subject: [PATCH 181/194] change message format for TesExceptionHandler --- tes-adapter/scripts/application.properties | 2 +- .../tesadapter/controller/TesExceptionHandler.java | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tes-adapter/scripts/application.properties b/tes-adapter/scripts/application.properties index bcd3b0fbc3..2818e9039e 100644 --- a/tes-adapter/scripts/application.properties +++ b/tes-adapter/scripts/application.properties @@ -1,4 +1,4 @@ -cloud.pipeline.host=https://${CP_API_SRV_INTERNAL_HOST:cp-api-srv.default.svc.cluster.local}:${CP_API_SRV_INTERNAL_PORT:31080}/pipeline/ +cloud.pipeline.host=https://${CP_API_SRV_INTERNAL_HOST:cp-api-srv.default.svc.cluster.local}:${CP_API_SRV_INTERNAL_PORT:31080}/pipeline/restapi/ cloud.pipeline.token=${CP_API_JWT_ADMIN} diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java index 5f9867fdae..123bc036d5 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java @@ -13,15 +13,22 @@ @RestControllerAdvice() public class TesExceptionHandler { - @Autowired + private static final String ERROR_MESSAGE_FORMAT = "%s\n%s"; + private MessageHelper messageHelper; + @Autowired + public TesExceptionHandler(final MessageHelper messageHelper) { + this.messageHelper = messageHelper; + } + @ExceptionHandler(Throwable.class) public final ResponseEntity handleUncaughtException(final Throwable exception, final WebRequest request) { log.error(messageHelper.getMessage("logger.error", request.getDescription(true)), exception); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) - .body(exception.getMessage() + request.getDescription(true)); + .body(String.format(ERROR_MESSAGE_FORMAT, exception.getMessage(), request.getDescription(true))); } + } From 2f41dfd6701a3181fd9e885a8e68f754b971efe5 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Wed, 9 Oct 2019 14:50:47 +0300 Subject: [PATCH 182/194] refactoring cp-tes-erv-dpl.yaml --- deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml index 93e4356a8c..70225f016b 100644 --- a/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml +++ b/deploy/contents/k8s/cp-tes-srv/cp-tes-srv-dpl.yaml @@ -20,7 +20,7 @@ spec: containers: - name: cp-tes-srv image: lifescience/cloud-pipeline:tes-srv-$CP_VERSION - imagePullPolicy: "ifNotPresent" + imagePullPolicy: "IfNotPresent" ports: - containerPort: 8080 envFrom: From d95d085d56f045446d2b0334805bf3d253590a35 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 9 Oct 2019 15:47:54 +0300 Subject: [PATCH 183/194] Added "Bearer" filter for token-check preHandle method in TesTokenInterceptor --- .../controller/TesTokenInterceptor.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index f5ec06046e..879bdd1013 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -26,6 +26,8 @@ public class TesTokenInterceptor implements HandlerInterceptor { private static final String HTTP_AUTH_COOKIE = "HttpAuthorization"; + private static final String BEARER_PREFIX = "Bearer "; + private static final String EMPTY_PREFIX = ""; private TesTokenHolder tesTokenHolder; @@ -68,11 +70,22 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons private Optional checkRequestForToken(HttpServletRequest request) { if (StringUtils.isNotEmpty(request.getHeader(HttpHeaders.AUTHORIZATION))) { - return Optional.of(request.getHeader(HttpHeaders.AUTHORIZATION)); + if (request.getHeader(HttpHeaders.AUTHORIZATION).startsWith(BEARER_PREFIX)) { + return Optional.of(request.getHeader(HttpHeaders.AUTHORIZATION) + .replaceFirst(BEARER_PREFIX, EMPTY_PREFIX)); + } else { + return Optional.of(request.getHeader(HttpHeaders.AUTHORIZATION)); + } } else if (ArrayUtils.isNotEmpty(request.getCookies())) { return Arrays.stream(request.getCookies()).filter(cookie -> cookie.getName().equalsIgnoreCase(HTTP_AUTH_COOKIE)) - .map(Cookie::getValue).findFirst(); + .map(Cookie::getValue).findFirst().map(cookieToken -> { + if (cookieToken.startsWith(BEARER_PREFIX)) { + return cookieToken.replaceFirst(BEARER_PREFIX, EMPTY_PREFIX); + } else { + return cookieToken; + } + }); } return Optional.empty(); } From fda6da5c52d14969a9bf3432023b6b910e3a97b6 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Wed, 9 Oct 2019 15:50:25 +0300 Subject: [PATCH 184/194] Added config dir for DOckerfile --- tes-adapter/scripts/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tes-adapter/scripts/Dockerfile b/tes-adapter/scripts/Dockerfile index f4401386b2..8d3818b086 100644 --- a/tes-adapter/scripts/Dockerfile +++ b/tes-adapter/scripts/Dockerfile @@ -24,7 +24,8 @@ RUN chmod +x /init # start script RUN ["/init", "tes-support", "https://github.com/SilinPavel/cloud-pipeline.git"] # copy application.properties -COPY application.properties / +RUN mkdir config +COPY application.properties config/ # run jar CMD ["java", "-jar", "tes-adapter-1.0-SNAPSHOT.jar"] # expose port From 4f1fd73403d6defd6ecc90b749bfda361df31843 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Fri, 18 Oct 2019 18:35:52 +0300 Subject: [PATCH 185/194] add CP_TES_WHITE_IPV4_CIDR into properties and describes into readme --- deploy/README.md | 3 +++ deploy/contents/install/install-config | 3 +++ tes-adapter/scripts/application.properties | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/deploy/README.md b/deploy/README.md index e27883ad7c..0dafe3d59c 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -125,6 +125,9 @@ bash build.sh -aws eu-central-1,us-east-1 \ # List of re -env CP_KUBE_MASTER_ETCD_HOST_PATH= \ # Allows to override a location of the folder where etcd stores wal/data dirs. This is useful when etcd runs into I/O latency issues and shall be pointed to another device mounted to a more custom location, which leads to the kube control plane failures. If not defined - /var/lib/etcd path will be used -env CP_KUBE_MIN_DNS_REPLICAS= \ # Allows to configure a minimal number of DNS replicas for the cluster (default: 1). DNS will be autoscaled based on the size of a cluster (1 new replica for each 128 cores or 5 nodes) -env CP_KUBE_SERVICES_TYPE= \ # Allows to select a preferred services mode type: "node-port" or "external-ip" (default: "node-port") + + # TES Adapter + -env CP_TES_WHITE_IPV4_CIDR # Whitelist ip addresses for access to Cloud Pipeline through TES adapter, if client doesn't have HttpAuthorization in headers or cookies. It supports as IPv4 as Ipv6 ``` # Examples diff --git a/deploy/contents/install/install-config b/deploy/contents/install/install-config index a5f62d41fb..91f3aa51e3 100644 --- a/deploy/contents/install/install-config +++ b/deploy/contents/install/install-config @@ -154,5 +154,8 @@ CP_DAV_SERVE_DIR=/dav-serve CP_DAV_MOUNT_POINT=/dav-mount CP_DAV_INSTRUCTIONS_URL= +# cp-tes-srv +CP_TES_WHITE_IPV4_CIDR=127.0.0.1/24 + # Other params, specified on command line # diff --git a/tes-adapter/scripts/application.properties b/tes-adapter/scripts/application.properties index 2818e9039e..351d6f23ba 100644 --- a/tes-adapter/scripts/application.properties +++ b/tes-adapter/scripts/application.properties @@ -1,4 +1,4 @@ cloud.pipeline.host=https://${CP_API_SRV_INTERNAL_HOST:cp-api-srv.default.svc.cluster.local}:${CP_API_SRV_INTERNAL_PORT:31080}/pipeline/restapi/ cloud.pipeline.token=${CP_API_JWT_ADMIN} - +security.allowed.client.ipv4.range=${CP_TES_WHITE_IPV4_CIDR} From b3e18c34fb60a7840b42ef9e25610d48a6c762cd Mon Sep 17 00:00:00 2001 From: pavel_silin Date: Thu, 24 Oct 2019 12:02:28 +0300 Subject: [PATCH 186/194] add Dockerfile to deploy dir --- deploy/docker/cp-tes-srv/Dockerfile | 34 +++++++++++- .../cp-tes-srv/config/application.properties | 4 ++ deploy/docker/cp-tes-srv/get_jar-file.sh | 54 +++++++++++++++++++ tes-adapter/scripts/Dockerfile | 3 +- 4 files changed, 93 insertions(+), 2 deletions(-) create mode 100755 deploy/docker/cp-tes-srv/get_jar-file.sh diff --git a/deploy/docker/cp-tes-srv/Dockerfile b/deploy/docker/cp-tes-srv/Dockerfile index e545f889c1..8e87c33630 100644 --- a/deploy/docker/cp-tes-srv/Dockerfile +++ b/deploy/docker/cp-tes-srv/Dockerfile @@ -12,4 +12,36 @@ # See the License for the specific language governing permissions and # limitations under the License. -# FIXME Docker implementation should be here +# start from base +FROM centos:7 +# install tools +RUN yum install -y \ + git \ + wget \ + unzip +# url jdk 1.8.0 +ARG JAVA_SOURCE="https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u222-b10/OpenJDK8U-jdk_x64_linux_hotspot_8u222b10.tar.gz" +# set /usr/lib as WORKDIR +WORKDIR /usr/lib +# download jdk and unzip it +RUN wget -P /usr/lib/ $JAVA_SOURCE && \ + tar -zxf OpenJDK8U-jdk_x64_linux_hotspot_8u222b10.tar.gz && \ + rm -f OpenJDK8U-jdk_x64_linux_hotspot_8u222b10.tar.gz +# set JAVA_HOME enviroment +ENV JAVA_HOME="/usr/lib/jdk8u222-b10/" +# set PATH enviroment +ENV PATH="$JAVA_HOME/bin:${PATH}" +# copy script +COPY get_jar-file.sh /init +# change rule +RUN chmod +x /init +WORKDIR /opt +# start script +RUN ["/init", "tes-dev", "https://github.com/SilinPavel/cloud-pipeline.git"] +# copy application.properties +RUN mkdir config +COPY config/application.properties config/ +# run jar +CMD ["java", "-jar", "tes-adapter-1.0-SNAPSHOT.jar"] +# expose port +EXPOSE 8080 \ No newline at end of file diff --git a/deploy/docker/cp-tes-srv/config/application.properties b/deploy/docker/cp-tes-srv/config/application.properties index e69de29bb2..351d6f23ba 100644 --- a/deploy/docker/cp-tes-srv/config/application.properties +++ b/deploy/docker/cp-tes-srv/config/application.properties @@ -0,0 +1,4 @@ +cloud.pipeline.host=https://${CP_API_SRV_INTERNAL_HOST:cp-api-srv.default.svc.cluster.local}:${CP_API_SRV_INTERNAL_PORT:31080}/pipeline/restapi/ +cloud.pipeline.token=${CP_API_JWT_ADMIN} +security.allowed.client.ipv4.range=${CP_TES_WHITE_IPV4_CIDR} + diff --git a/deploy/docker/cp-tes-srv/get_jar-file.sh b/deploy/docker/cp-tes-srv/get_jar-file.sh new file mode 100755 index 0000000000..20667cd969 --- /dev/null +++ b/deploy/docker/cp-tes-srv/get_jar-file.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# exit when any command fails +set -e +# keep track of the last executed command +trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG +# echo an error message before exiting +trap 'echo "\"${last_command}\" command filed with exit code $?." cleanup' ERR + +# deletes the temp directory +function cleanup { + rm -rf $tmpdir + echo "Deleted temp directory $tmpdir" +} + +#cleanup before exiting +trap cleanup EXIT + +workdir=$(pwd) + +if [ -n "$2" ]; then + gitRepoUrl="-b $1 $2" +else + gitRepoUrl=$1 +fi + +echo "Creating tmp directory..." + +tmpdir=$(mktemp -d -t cloud_pipeline-XXXXXXXXXX) +TES_PATH=$tmpdir/tes-adapter +TES_JAR_PATH=$TES_PATH/build/libs/tes-adapter-1.0-SNAPSHOT.jar + +# Step 1. +# Cloning cloud-pipeline(CP) API from git repository, +# where $1 is URL(with possible specified branch(-b)), and +# $2 is where the CP will be copied. +echo "Cloning git-repo from $gitRepoUrl" +git clone $gitRepoUrl $tmpdir; + +# Step 2. +# Build jar file from tes-adapter module(TES-PATH) +echo "Building jar-file..." +if [ -d "$TES_PATH" ]; then + cd $tmpdir + ./gradlew tes-adapter:bootJar >/dev/null 2>&1 +else + echo "The $TES_PATH directory doesn't exist!" 2>&1; exit 1 +fi + +# Step 3 +echo "Copying jar-file from tes-adapter to $workdir" +cp -rf $TES_JAR_PATH $workdir + +echo "Success! You can find the corresponding jar-file in the folder $workdir" \ No newline at end of file diff --git a/tes-adapter/scripts/Dockerfile b/tes-adapter/scripts/Dockerfile index 8d3818b086..321df1caf1 100644 --- a/tes-adapter/scripts/Dockerfile +++ b/tes-adapter/scripts/Dockerfile @@ -21,8 +21,9 @@ ENV PATH="$JAVA_HOME/bin:${PATH}" COPY get_jar-file.sh /init # change rule RUN chmod +x /init +WORKDIR /opt # start script -RUN ["/init", "tes-support", "https://github.com/SilinPavel/cloud-pipeline.git"] +RUN ["/init", "tes-dev", "https://github.com/SilinPavel/cloud-pipeline.git"] # copy application.properties RUN mkdir config COPY application.properties config/ From 6ad68daaeb63dbba513a143f48918197de85fd50 Mon Sep 17 00:00:00 2001 From: pavel_silin Date: Thu, 24 Oct 2019 12:44:12 +0300 Subject: [PATCH 187/194] do not send 401 from /error page --- .../pipeline/tesadapter/controller/TesTokenInterceptor.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index 879bdd1013..d56d199ba5 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -28,6 +28,7 @@ public class TesTokenInterceptor implements HandlerInterceptor { private static final String HTTP_AUTH_COOKIE = "HttpAuthorization"; private static final String BEARER_PREFIX = "Bearer "; private static final String EMPTY_PREFIX = ""; + private static final String ERROR_PATH = "/error"; private TesTokenHolder tesTokenHolder; @@ -53,6 +54,9 @@ public TesTokenInterceptor(TesTokenHolder tesTokenHolder, @Override public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { + if (request.getServletPath().equalsIgnoreCase(ERROR_PATH)) { + return true; + } final Optional requestToken = checkRequestForToken(request); if (requestToken.isPresent()) { log.debug(messageHelper.getMessage(MessageConstants.TOKEN_FOUND_IN_REQUEST, request.getServletPath())); From 108f696fcd223c25d6425978652123af0dffd0d6 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 25 Oct 2019 10:52:22 +0300 Subject: [PATCH 188/194] couple refactoring of swagger-page titles --- .../pipeline/tesadapter/configuration/TesSwaggerConfig.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java index 1578feb819..6625f5c701 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java @@ -26,10 +26,8 @@ public Docket api() { private ApiInfo getApiInfo() { return new ApiInfoBuilder() - .title("TES adapter API Documentation") - .description("Simple TES adapter application. " + - "Without Security Configuration and Context") - .version("1.0.0") + .title("Task Execution Service (TES) API") + .description("Describes the REST API provided by a Cloud-Pipeline Task Execution Service") .build(); } } From abf0ed0ee81caf400928380dda37f579d1d0d2d2 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Fri, 25 Oct 2019 11:07:48 +0300 Subject: [PATCH 189/194] couple changes of swagger-page titles --- .../pipeline/tesadapter/configuration/TesSwaggerConfig.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java index 6625f5c701..f560dd91f6 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/TesSwaggerConfig.java @@ -26,8 +26,9 @@ public Docket api() { private ApiInfo getApiInfo() { return new ApiInfoBuilder() - .title("Task Execution Service (TES) API") - .description("Describes the REST API provided by a Cloud-Pipeline Task Execution Service") + .title("Task Execution Service (GA4GH TES) API") + .description("Represents implementation of GA4GH TES " + + "(https://github.com/ga4gh/task-execution-schemas) API for Cloud Pipeline") .build(); } } From c89c6c219a169fc7479b0462f9728698840d52b4 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 29 Oct 2019 18:39:28 +0300 Subject: [PATCH 190/194] I fixed HttpMessageConverter to it doesn't have null fields and refactoring code. Now code has 1 failed test --- .../configuration/RestConfiguration.java | 6 ++++-- .../controller/TesExceptionHandler.java | 1 - .../controller/TesTokenInterceptor.java | 9 +++------ .../service/CloudPipelineAPIClient.java | 5 ++--- .../pipeline/tesadapter/service/TaskMapper.java | 17 ++++++++--------- .../tesadapter/service/TesTaskServiceImpl.java | 12 ++++++------ .../app/TesAdapterControllerTest.java | 4 +++- 7 files changed, 26 insertions(+), 28 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java index 65910d5d64..62718f18a1 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/configuration/RestConfiguration.java @@ -2,13 +2,14 @@ import com.epam.pipeline.tesadapter.controller.TesTokenInterceptor; import com.epam.pipeline.tesadapter.entity.TesTokenHolder; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.http.HttpMessageConverters; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Scope; import org.springframework.context.annotation.ScopedProxyMode; -import org.springframework.http.converter.HttpMessageConverter; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; import org.springframework.web.context.WebApplicationContext; import org.springframework.web.servlet.config.annotation.InterceptorRegistry; @@ -22,7 +23,8 @@ public class RestConfiguration implements WebMvcConfigurer { @Bean public HttpMessageConverters customConverters() { - HttpMessageConverter addition = new MappingJackson2HttpMessageConverter(); + MappingJackson2HttpMessageConverter addition = new MappingJackson2HttpMessageConverter(); + addition.setObjectMapper(new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL)); return new HttpMessageConverters(false, Collections.singletonList(addition)); } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java index 123bc036d5..27fda2cb04 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java @@ -12,7 +12,6 @@ @Slf4j @RestControllerAdvice() public class TesExceptionHandler { - private static final String ERROR_MESSAGE_FORMAT = "%s\n%s"; private MessageHelper messageHelper; diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index d56d199ba5..d2d4e3a0d0 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -24,22 +24,19 @@ @SuppressWarnings("unused") @Component public class TesTokenInterceptor implements HandlerInterceptor { - private static final String HTTP_AUTH_COOKIE = "HttpAuthorization"; private static final String BEARER_PREFIX = "Bearer "; private static final String EMPTY_PREFIX = ""; private static final String ERROR_PATH = "/error"; + private final IpAddressMatcher ipAddressMatcherV4; + private final IpAddressMatcher ipAddressMatcherV6; private TesTokenHolder tesTokenHolder; + private MessageHelper messageHelper; @Value("${cloud.pipeline.token}") private String defaultPipelineToken; - private MessageHelper messageHelper; - - private final IpAddressMatcher ipAddressMatcherV4; - private final IpAddressMatcher ipAddressMatcherV6; - @Autowired public TesTokenInterceptor(TesTokenHolder tesTokenHolder, MessageHelper messageHelper, diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index 9fada94e56..1d48b4684a 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -28,12 +28,11 @@ @Service public class CloudPipelineAPIClient { - private TesTokenHolder tesTokenHolder; + private static final String CLOUD_PIPELINE_HOST = "cloudPipelineHost"; + private TesTokenHolder tesTokenHolder; private String cloudPipelineHostUrl; - private static final String CLOUD_PIPELINE_HOST = "cloudPipelineHost"; - @Autowired public CloudPipelineAPIClient(TesTokenHolder tesTokenHolder, MessageHelper messageHelper, diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java index b172f0d833..95653b11a4 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TaskMapper.java @@ -44,15 +44,6 @@ @Slf4j @Service public class TaskMapper { - - private final Integer defaultHddSize; - private final Double defaultRamGb; - private final Long defaultCpuCore; - private final Boolean defaultPreemptible; - private final String defaultRegion; - private MessageHelper messageHelper; - private final CloudPipelineAPIClient cloudPipelineAPIClient; - private static final String SEPARATOR = " "; private static final Integer DEFAULT_HDD_SIZE = 30; private static final Double DEFAULT_RAM_GB = 4.0; @@ -81,6 +72,14 @@ public class TaskMapper { private static final Double PIB_TO_GIB = 1048576.0; private static final Double EIB_TO_GIB = 1073741824.0; + private final Integer defaultHddSize; + private final Double defaultRamGb; + private final Long defaultCpuCore; + private final Boolean defaultPreemptible; + private final String defaultRegion; + private final CloudPipelineAPIClient cloudPipelineAPIClient; + private MessageHelper messageHelper; + @Autowired public TaskMapper(@Value("${cloud.pipeline.hddSize}") Integer hddSize, @Value("${cloud.pipeline.ramGb}") Double defaultRamGb, diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 522f99b3f0..f67403c1a9 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -33,12 +33,6 @@ @Slf4j @Service public class TesTaskServiceImpl implements TesTaskService { - - private String nameOfService; - private String doc; - private final CloudPipelineAPIClient cloudPipelineAPIClient; - private final TaskMapper taskMapper; - private final MessageHelper messageHelper; private static final TaskView DEFAULT_TASK_VIEW = TaskView.MINIMAL; private static final String ID = "id"; private static final String NAME_PREFIX = "pod.id"; @@ -48,6 +42,12 @@ public class TesTaskServiceImpl implements TesTaskService { private static final Boolean LOAD_STORAGE_LINKS = true; private static final Long DEFAULT_PAGE_SIZE = 256L; + private final CloudPipelineAPIClient cloudPipelineAPIClient; + private final TaskMapper taskMapper; + private final MessageHelper messageHelper; + private String nameOfService; + private String doc; + @Autowired public TesTaskServiceImpl(@Value("${cloud.pipeline.service.name}") String nameOfService, @Value("${cloud.pipeline.doc}") String doc, diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 9d2c9d526a..e1c4886929 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -13,6 +13,7 @@ import com.epam.pipeline.tesadapter.entity.TesTask; import com.epam.pipeline.tesadapter.service.CloudPipelineAPIClient; import com.epam.pipeline.tesadapter.service.TesTaskServiceImpl; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -164,7 +165,8 @@ public void getTesTaskWhenRequestingReturnTesTaskResponse() throws Exception { .header(HttpHeaders.AUTHORIZATION, defaultPipelineToken) .contentType(JSON_CONTENT_TYPE)) .andDo(print()).andExpect(status().isOk()).andExpect(content() - .json(new ObjectMapper().writeValueAsString(tesTask))); + .json(new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL) + .writeValueAsString(tesTask))); } @Test From be09cedfb39efe2f8cd317bb6df55e58c37f3842 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 29 Oct 2019 19:53:52 +0300 Subject: [PATCH 191/194] I fixed HttpMessageConverter to it doesn't have null fields and refactoring code. Fixed TesAdapterControllerTest --- tes-adapter/src/main/resources/messages.properties | 2 +- .../pipeline/tesadapter/app/TesAdapterControllerTest.java | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tes-adapter/src/main/resources/messages.properties b/tes-adapter/src/main/resources/messages.properties index 113bbc914d..80cb791b9e 100644 --- a/tes-adapter/src/main/resources/messages.properties +++ b/tes-adapter/src/main/resources/messages.properties @@ -5,7 +5,7 @@ error.parameter.required=Parameter ''{0}'' id required for ''{1}'' podId. #Parameters error.parameter.non.scalar.type=Unable to resolve parameter value ''{0}''. Reference value for field ''{1}'' is used as a parameter value. -error.parameter.incompatible.content=Parameter (id ''{0}'') with undesirable content. +error.parameter.incompatible.content=Parameter (id ''{0}'') with undesirable content.uri=/v1/tasks/%20:cancel;client=127.0.0.1 debug.token.found.in.request=Auth token found in HEADERS or COOKIE for request ''{0}'' debug.ip.is.accepted=Get default auth token, since request host is from accepted ip range for request ''{0}'' diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index e1c4886929..0bb2937291 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -56,7 +56,6 @@ public class TesAdapterControllerTest { private static final TaskView DEFAULT_VIEW = TaskView.MINIMAL; private static final String STUBBED_SUBMIT_JSON_REQUEST = "{}"; private static final String STUBBED_SUBMIT_JSON_RESPONSE = "{\"id\":\"5\"}"; - private static final String CANCEL_REQUEST_DESCRIPTION = "uri=/v1/tasks/%20:cancel;client=127.0.0.1"; private static final String JSON_CONTENT_TYPE = "application/json"; private static final String HTTP_AUTH_COOKIE = "HttpAuthorization"; private static final String WRONG_TOKEN = "wrongPipelineToken"; @@ -143,8 +142,7 @@ public void expectIllegalStateExceptionWhenRunCancelTesTaskWithWrongId() throws .andDo(print()) .andExpect(status().isInternalServerError()) .andExpect(content().string(containsString(messageHelper - .getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, "taskId") - + CANCEL_REQUEST_DESCRIPTION))); + .getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT,"taskId")))); } @Test From 6ddbac850c7c1a5642bb3bd96f1cc031ef4d67d7 Mon Sep 17 00:00:00 2001 From: Ilya Ugryumov Date: Tue, 29 Oct 2019 19:54:40 +0300 Subject: [PATCH 192/194] refactor --- .../epam/pipeline/tesadapter/app/TesAdapterControllerTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 0bb2937291..cab289b874 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -1,6 +1,5 @@ package com.epam.pipeline.tesadapter.app; - import com.epam.pipeline.tesadapter.common.MessageConstants; import com.epam.pipeline.tesadapter.common.MessageHelper; import com.epam.pipeline.tesadapter.controller.TesAdapterController; @@ -39,7 +38,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - @WebMvcTest(TesAdapterController.class) @SuppressWarnings({"unused", "PMD.TooManyStaticImports", "PMD.AvoidUsingHardCodedIP", "PMD.JUnitTestsShouldIncludeAssert"}) From 38de490e86d0d0702be6c1bb25bc1b34d6aada34 Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 29 Oct 2019 21:55:24 +0300 Subject: [PATCH 193/194] couple clean-code and refactoring of whole sub-project. Added nextPageToken for listTesTask method response --- .../tesadapter/common/MessageConstants.java | 2 -- .../tesadapter/common/MessageHelper.java | 2 -- .../controller/TesExceptionHandler.java | 3 -- .../controller/TesTokenInterceptor.java | 5 +-- .../entity/TesListTasksResponse.java | 2 ++ .../service/CloudPipelineAPIClient.java | 5 ++- .../service/TesTaskServiceImpl.java | 33 +++++++++++-------- .../src/main/resources/application.properties | 4 +-- .../src/main/resources/log4j2-spring.xml | 2 +- .../src/main/resources/messages.properties | 5 +-- .../app/TesAdapterControllerTest.java | 11 +------ .../tesadapter/service/TaskMapperTest.java | 29 ++++++---------- .../service/TesTaskServiceImplTest.java | 32 +++++++----------- .../resources/test-application.properties | 2 +- 14 files changed, 50 insertions(+), 87 deletions(-) diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java index 6991d8fa7b..8bb1a503a9 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageConstants.java @@ -2,7 +2,6 @@ @SuppressWarnings({"PMD.UseUtilityClass", "checkstyle:HideUtilityClassConstructor"}) public final class MessageConstants { - //Common errors public static final String ERROR_PARAMETER_REQUIRED = "error.parameter.required"; public static final String ERROR_PARAMETER_NULL_OR_EMPTY = "error.null.param"; @@ -21,5 +20,4 @@ public final class MessageConstants { public static final String CANCEL_PIPELINE_RUN_BY_ID = "debug.cancel.pipeline.run.by.id"; public static final String GET_PIPELINE_RUN_BY_ID = "debug.get.pipeline.run.by.id"; public static final String GET_SERVICE_INFO = "debug.get.service.info"; - } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageHelper.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageHelper.java index efc17b2ee6..58cf7cb90e 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageHelper.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/common/MessageHelper.java @@ -23,7 +23,6 @@ @Component public class MessageHelper { - private MessageSource messageSource; public MessageHelper(final MessageSource messageSource) { @@ -50,5 +49,4 @@ public MessageSource getMessageSource() { public String getMessage(final String code, final Object... args) { return getMessageSource().getMessage(code, args, code, Locale.getDefault()); } - } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java index 123bc036d5..740139b1d1 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesExceptionHandler.java @@ -12,7 +12,6 @@ @Slf4j @RestControllerAdvice() public class TesExceptionHandler { - private static final String ERROR_MESSAGE_FORMAT = "%s\n%s"; private MessageHelper messageHelper; @@ -28,7 +27,5 @@ public final ResponseEntity handleUncaughtException(final Throwable exce log.error(messageHelper.getMessage("logger.error", request.getDescription(true)), exception); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) .body(String.format(ERROR_MESSAGE_FORMAT, exception.getMessage(), request.getDescription(true))); - } - } diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java index d56d199ba5..82babec37a 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/controller/TesTokenInterceptor.java @@ -24,19 +24,16 @@ @SuppressWarnings("unused") @Component public class TesTokenInterceptor implements HandlerInterceptor { - private static final String HTTP_AUTH_COOKIE = "HttpAuthorization"; private static final String BEARER_PREFIX = "Bearer "; private static final String EMPTY_PREFIX = ""; private static final String ERROR_PATH = "/error"; - private TesTokenHolder tesTokenHolder; - @Value("${cloud.pipeline.token}") private String defaultPipelineToken; + private TesTokenHolder tesTokenHolder; private MessageHelper messageHelper; - private final IpAddressMatcher ipAddressMatcherV4; private final IpAddressMatcher ipAddressMatcherV6; diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java index a82c5d8899..7a8ce096e0 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/entity/TesListTasksResponse.java @@ -3,12 +3,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.Builder; import lombok.Data; import lombok.NonNull; import javax.validation.Valid; import java.util.List; +@Builder @ApiModel(description = "ListTasksResponse describes a response from the ListTasks endpoint.") @Data public class TesListTasksResponse { diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java index 9fada94e56..1d48b4684a 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/CloudPipelineAPIClient.java @@ -28,12 +28,11 @@ @Service public class CloudPipelineAPIClient { - private TesTokenHolder tesTokenHolder; + private static final String CLOUD_PIPELINE_HOST = "cloudPipelineHost"; + private TesTokenHolder tesTokenHolder; private String cloudPipelineHostUrl; - private static final String CLOUD_PIPELINE_HOST = "cloudPipelineHost"; - @Autowired public CloudPipelineAPIClient(TesTokenHolder tesTokenHolder, MessageHelper messageHelper, diff --git a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java index 522f99b3f0..0da3cc1961 100644 --- a/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java +++ b/tes-adapter/src/main/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImpl.java @@ -42,7 +42,7 @@ public class TesTaskServiceImpl implements TesTaskService { private static final TaskView DEFAULT_TASK_VIEW = TaskView.MINIMAL; private static final String ID = "id"; private static final String NAME_PREFIX = "pod.id"; - private static final String DEFAULT_PAGE_TOKEN = "1"; + private static final Integer DEFAULT_PAGE_TOKEN = 1; private static final String DEFAULT_NAME_SERVICE = "CloudPipeline"; private static final String DEFAULT_DOC = " "; private static final Boolean LOAD_STORAGE_LINKS = true; @@ -73,9 +73,11 @@ public TesCreateTaskResponse submitTesTask(TesTask body) { @Override public TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String pageToken, TaskView view) { + final int pageNumber = Optional.ofNullable(pageToken).map(p -> parseStringNumber(p).intValue()) + .orElse(DEFAULT_PAGE_TOKEN); final List tesTaskList = (StringUtils.isNotEmpty(namePrefix) - ? searchRunsWithNamePrefix(namePrefix, pageSize, pageToken) - : filterRunsWithOutNamePrefix(pageSize, pageToken) + ? searchRunsWithNamePrefix(namePrefix, pageSize, pageNumber) + : filterRunsWithOutNamePrefix(pageSize, pageNumber) ).stream() .map(pipelineRun -> taskMapper.mapToTesTask( @@ -83,10 +85,13 @@ public TesListTasksResponse listTesTask(String namePrefix, Long pageSize, String Optional.ofNullable(view).orElse(DEFAULT_TASK_VIEW) )) .collect(Collectors.toList()); - return new TesListTasksResponse(tesTaskList); + return TesListTasksResponse.builder() + .tasks(tesTaskList) + .nextPageToken(String.valueOf(pageNumber + 1)) + .build(); } - private List searchRunsWithNamePrefix(String namePrefix, Long pageSize, String pageToken) { + private List searchRunsWithNamePrefix(String namePrefix, Long pageSize, Integer pageNumber) { PagingRunFilterExpressionVO filterExpressionVO = new PagingRunFilterExpressionVO(); FilterExpressionVO expression = new FilterExpressionVO(); expression.setField(NAME_PREFIX); @@ -94,15 +99,15 @@ private List searchRunsWithNamePrefix(String namePrefix, Long pageS expression.setOperand(FilterOperandTypeVO.EQUALS.getOperand()); expression.setFilterExpressionType(FilterExpressionTypeVO.LOGICAL); filterExpressionVO.setFilterExpression(expression); - filterExpressionVO.setPage(Integer.parseInt(Optional.ofNullable(pageToken).orElse(DEFAULT_PAGE_TOKEN))); + filterExpressionVO.setPage(pageNumber); filterExpressionVO.setPageSize(Optional.ofNullable(pageSize).orElse(DEFAULT_PAGE_SIZE).intValue()); log.debug(messageHelper.getMessage(MessageConstants.GET_LIST_TASKS_BY_NAME_PREFIX, namePrefix)); return ListUtils.emptyIfNull(cloudPipelineAPIClient.searchRuns(filterExpressionVO).getElements()); } - private List filterRunsWithOutNamePrefix(Long pageSize, String pageToken) { + private List filterRunsWithOutNamePrefix(Long pageSize, Integer pageNumber) { PagingRunFilterVO filterVO = new PagingRunFilterVO(); - filterVO.setPage(Integer.parseInt(Optional.ofNullable(pageToken).orElse(DEFAULT_PAGE_TOKEN))); + filterVO.setPage(pageNumber); filterVO.setPageSize(Optional.ofNullable(pageSize).orElse(DEFAULT_PAGE_SIZE).intValue()); log.debug(messageHelper.getMessage(MessageConstants.GET_LIST_TASKS_BY_DEFAULT_PREFIX)); return ListUtils.emptyIfNull(cloudPipelineAPIClient.filterRuns(filterVO, LOAD_STORAGE_LINKS).getElements()); @@ -112,7 +117,7 @@ private List filterRunsWithOutNamePrefix(Long pageSize, String page public TesCancelTaskResponse cancelTesTask(String id) { RunStatusVO updateStatus = new RunStatusVO(); updateStatus.setStatus(TaskStatus.STOPPED); - cloudPipelineAPIClient.updateRunStatus(parseRunId(id), updateStatus); + cloudPipelineAPIClient.updateRunStatus(parseStringNumber(id), updateStatus); log.debug(messageHelper.getMessage(MessageConstants.CANCEL_PIPELINE_RUN_BY_ID, id)); return new TesCancelTaskResponse(); } @@ -120,13 +125,13 @@ public TesCancelTaskResponse cancelTesTask(String id) { @Override public TesTask getTesTask(String id, TaskView view) { log.debug(messageHelper.getMessage(MessageConstants.GET_PIPELINE_RUN_BY_ID, id)); - return taskMapper.mapToTesTask(cloudPipelineAPIClient.loadPipelineRun(parseRunId(id)), view); + return taskMapper.mapToTesTask(cloudPipelineAPIClient.loadPipelineRun(parseStringNumber(id)), view); } - private Long parseRunId(String id) { - Assert.state(StringUtils.isNumeric(id), - messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, ID)); - return Long.parseLong(id); + private Long parseStringNumber(String number) { + Assert.state(StringUtils.isNumeric(number), + messageHelper.getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, number)); + return Long.parseLong(number); } @Override diff --git a/tes-adapter/src/main/resources/application.properties b/tes-adapter/src/main/resources/application.properties index 649f97cbec..a6706ffafb 100644 --- a/tes-adapter/src/main/resources/application.properties +++ b/tes-adapter/src/main/resources/application.properties @@ -9,6 +9,4 @@ cloud.pipeline.region=eu-central-1 cloud.pipeline.service.name=CloudPipeline cloud.pipeline.doc=https://epam.github.io/cloud-pipeline/ security.allowed.client.ipv4.range=127.0.0.1/24 -security.allowed.client.ipv6.range=::1/128 - - +security.allowed.client.ipv6.range=::1/128 \ No newline at end of file diff --git a/tes-adapter/src/main/resources/log4j2-spring.xml b/tes-adapter/src/main/resources/log4j2-spring.xml index 9ee48e1150..e68faed91c 100644 --- a/tes-adapter/src/main/resources/log4j2-spring.xml +++ b/tes-adapter/src/main/resources/log4j2-spring.xml @@ -80,4 +80,4 @@ - + \ No newline at end of file diff --git a/tes-adapter/src/main/resources/messages.properties b/tes-adapter/src/main/resources/messages.properties index 113bbc914d..424d383b10 100644 --- a/tes-adapter/src/main/resources/messages.properties +++ b/tes-adapter/src/main/resources/messages.properties @@ -1,11 +1,8 @@ #general error messages error.null.param=Error: the required parameter ''{0}'' is null or empty error.parameter.required=Parameter ''{0}'' id required for ''{1}'' podId. - - -#Parameters error.parameter.non.scalar.type=Unable to resolve parameter value ''{0}''. Reference value for field ''{1}'' is used as a parameter value. -error.parameter.incompatible.content=Parameter (id ''{0}'') with undesirable content. +error.parameter.incompatible.content=Parameter (id ''{0}'') with undesirable content.uri=/v1/tasks/%20:cancel;client=127.0.0.1 debug.token.found.in.request=Auth token found in HEADERS or COOKIE for request ''{0}'' debug.ip.is.accepted=Get default auth token, since request host is from accepted ip range for request ''{0}'' diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java index 9d2c9d526a..8f65752044 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/app/TesAdapterControllerTest.java @@ -11,7 +11,6 @@ import com.epam.pipeline.tesadapter.entity.TesListTasksResponse; import com.epam.pipeline.tesadapter.entity.TesServiceInfo; import com.epam.pipeline.tesadapter.entity.TesTask; -import com.epam.pipeline.tesadapter.service.CloudPipelineAPIClient; import com.epam.pipeline.tesadapter.service.TesTaskServiceImpl; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.BeforeEach; @@ -54,11 +53,8 @@ public class TesAdapterControllerTest { private static final Long PAGE_SIZE = 20L; private static final TaskView DEFAULT_VIEW = TaskView.MINIMAL; private static final String STUBBED_SUBMIT_JSON_REQUEST = "{}"; - private static final String STUBBED_SUBMIT_JSON_RESPONSE = "{\"id\":\"5\"}"; - private static final String CANCEL_REQUEST_DESCRIPTION = "uri=/v1/tasks/%20:cancel;client=127.0.0.1"; private static final String JSON_CONTENT_TYPE = "application/json"; private static final String HTTP_AUTH_COOKIE = "HttpAuthorization"; - private static final String WRONG_TOKEN = "wrongPipelineToken"; private static final String GET_SERVICE_INFO = "/v1/tasks/service-info"; private static final String IP_IN_RANGE = "192.168.1.1"; private static final String IP_OUT_OF_RANGE = "10.0.0.0"; @@ -81,9 +77,6 @@ public class TesAdapterControllerTest { @MockBean private TesTaskServiceImpl tesTaskService; - @MockBean - private CloudPipelineAPIClient cloudPipelineAPIClient; - private TesCreateTaskResponse tesCreateTaskResponse = new TesCreateTaskResponse(); private TesServiceInfo tesServiceInfo = new TesServiceInfo(); private TesTask tesTask = new TesTask(); @@ -91,7 +84,6 @@ public class TesAdapterControllerTest { @BeforeEach private void setUp() { - when(tesTaskService.cancelTesTask(DEFAULT_TASK_ID)).thenReturn(new TesCancelTaskResponse()); when(tesTaskService.listTesTask(NAME_PREFIX, PAGE_SIZE, PAGE_TOKEN, DEFAULT_VIEW)) .thenReturn(mock(TesListTasksResponse.class)); @@ -142,8 +134,7 @@ public void expectIllegalStateExceptionWhenRunCancelTesTaskWithWrongId() throws .andDo(print()) .andExpect(status().isInternalServerError()) .andExpect(content().string(containsString(messageHelper - .getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, "taskId") - + CANCEL_REQUEST_DESCRIPTION))); + .getMessage(MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, "taskId")))); } @Test diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java index fbec46ae51..8dd47344b8 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TaskMapperTest.java @@ -76,7 +76,7 @@ class TaskMapperTest { private static final String DEFAULT_COMMAND = "sleep 300"; private static final Long STUBBED_REGION_ID = 1L; private static final Long STUBBED_TOOL_ID = 11584L; - private static final Long TOOL_ID = 1l; + private static final Long TOOL_ID = 1L; private static final String STUBBED_IMAGE = "cp-docker-registry.default.svc.cluster.local:31443/library/centos:latest"; private static final String INPUT = "input"; private static final String OUTPUT = "output"; @@ -88,48 +88,39 @@ class TaskMapperTest { private static final String PIPELINE_TASK_NAME_CONSOLE = "Console"; private static final String PIPELINE_TASK_NAME_InitializeEnvironment = "InitializeEnvironment"; private static final String ANY_STRING = "any_string"; - private static List allowedInstanceTypes; private static final String GIB = "GiB"; private static final Double KIB_TO_GIB = 0.00000095367432; - private static final Double MIB_TO_GIB = 0.0009765625; - private static final Double GIB_TO_GIB = 1.0; - private static final Double TIB_TO_GIB = 1024.0; - private static final Double PIB_TO_GIB = 1048576.0; - private static final Double EIB_TO_GIB = 1073741824.0; + private static List allowedInstanceTypes; + private static TaskMapper taskMapper; + private static PipelineRun run = getPipelineRun(); + private static CloudPipelineAPIClient cloudPipelineAPIClient = mock(CloudPipelineAPIClient.class); @Autowired private MessageHelper messageHelper; - private static TaskMapper taskMapper; - private static PipelineRun run = getPipelineRun(); private List zones = new ArrayList<>(); private TesExecutor tesExecutor = new TesExecutor(); - private TesInput tesInput = mock(TesInput.class); - private TesOutput tesOutput = mock(TesOutput.class); private List tesExecutors = new ArrayList<>(); private List tesInputs = new ArrayList<>(); private List tesOutputs = new ArrayList<>(); private List abstractCloudRegions = new ArrayList<>(); - private AbstractCloudRegion abstractCloudRegion = mock(AbstractCloudRegion.class); - private static CloudPipelineAPIClient cloudPipelineAPIClient = mock(CloudPipelineAPIClient.class); - private AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = mock(AllowedInstanceAndPriceTypes.class); private Tool tool = mock(Tool.class); private TesTask tesTask = mock(TesTask.class); - private PipelineRun pipelineRun = mock(PipelineRun.class); - + private TesInput tesInput = mock(TesInput.class); + private TesOutput tesOutput = mock(TesOutput.class); + private AbstractCloudRegion abstractCloudRegion = mock(AbstractCloudRegion.class); + private AllowedInstanceAndPriceTypes allowedInstanceAndPriceTypes = mock(AllowedInstanceAndPriceTypes.class); @BeforeAll public static void setUpAll() { allowedInstanceTypes = new ArrayList<>(); fillWithAllowedInstanceTypes(allowedInstanceTypes); - AwsRegion awsRegion = new AwsRegion(); awsRegion.setRegionCode(DEFAULT_REGION_NAME); when(cloudPipelineAPIClient.loadRegion(anyLong())).thenReturn(awsRegion); - when(cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(TOOL_ID, 1l, + when(cloudPipelineAPIClient.loadAllowedInstanceAndPriceTypes(TOOL_ID, STUBBED_REGION_ID, true)).thenReturn(getAllowedInstanceAndPriceTypes(run)); - } @BeforeEach diff --git a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java index 6636f66b4f..dfa6468f92 100644 --- a/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java +++ b/tes-adapter/src/test/java/com/epam/pipeline/tesadapter/service/TesTaskServiceImplTest.java @@ -13,7 +13,6 @@ import com.epam.pipeline.vo.PagingRunFilterExpressionVO; import com.epam.pipeline.vo.PagingRunFilterVO; import com.epam.pipeline.vo.RunStatusVO; -import com.epam.pipeline.vo.filter.FilterExpressionVO; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -41,37 +40,28 @@ @ContextConfiguration(classes = {AppConfiguration.class}) @SuppressWarnings({"unused", "PMD.TooManyStaticImports"}) class TesTaskServiceImplTest { + private static final Integer NUMBER_OF_RUNS = 20; + private static final TaskView DEFAULT_TASK_VIEW = TaskView.MINIMAL; + private static final Long DEFAULT_PIPELINE_ID = 12410L; + private static final String ID = "id"; + private static final Boolean LOAD_STORAGE_LINKS = true; + @Value("${cloud.pipeline.service.name}") private String nameOfService; @Value("${cloud.pipeline.doc}") private String doc; - private TesTaskServiceImpl tesTaskService; - @Autowired private MessageHelper messageHelper; - private static final String DEFAULT_PAGE_TOKEN = "1"; - private static final String PREFIX_FIELD = "pod.id"; - private static final String PREFIX_NAME = "pipeline-12410"; - - private static final Integer NUMBER_OF_RUNS = 20; - private static final TaskView DEFAULT_TASK_VIEW = TaskView.MINIMAL; - private static final Long DEFAULT_PIPELINE_ID = 12410L; - private static final String ID = "id"; - private static final Integer FIRST = 0; - private static final Boolean LOAD_STORAGE_LINKS = true; - private static final Long DEFAULT_PAGE_SIZE = 256L; - - private List pipelineRunList = new ArrayList<>(); + private TesTaskServiceImpl tesTaskService; private TesTask tesTask = new TesTask(); - private PipelineStart pipelineStart = new PipelineStart(); private PipelineRun pipelineRun = new PipelineRun(); - private PagingRunFilterExpressionVO filterExpressionVO = new PagingRunFilterExpressionVO(); - private FilterExpressionVO expression = new FilterExpressionVO(); - + private PipelineStart pipelineStart = new PipelineStart(); + private List pipelineRunList = new ArrayList<>(); private PagedResult> pagedPipelineRunList = new PagedResult<>(pipelineRunList, NUMBER_OF_RUNS); + private TaskMapper taskMapper = mock(TaskMapper.class); private CloudPipelineAPIClient cloudPipelineAPIClient = mock(CloudPipelineAPIClient.class); @@ -118,7 +108,7 @@ public void expectIllegalStateExceptionWhenRunCancelTesTaskWithWrongId(String id IllegalStateException exception = assertThrows(IllegalStateException.class, () -> tesTaskService.cancelTesTask(id)); assertEquals(exception.getMessage(), messageHelper.getMessage( - MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, ID)); + MessageConstants.ERROR_PARAMETER_INCOMPATIBLE_CONTENT, id)); } private static Stream provideWrongIdInputForCancelTesTask() { diff --git a/tes-adapter/src/test/resources/test-application.properties b/tes-adapter/src/test/resources/test-application.properties index edea56b816..92088d9ace 100644 --- a/tes-adapter/src/test/resources/test-application.properties +++ b/tes-adapter/src/test/resources/test-application.properties @@ -4,4 +4,4 @@ cloud.pipeline.token=${API_TOKEN} cloud.pipeline.service.name=CloudPipeline cloud.pipeline.doc=https://epam.github.io/cloud-pipeline/ security.allowed.client.ipv4.range=192.168.1.0/24 -security.allowed.client.ipv6.range=::1/128 +security.allowed.client.ipv6.range=::1/128 \ No newline at end of file From 99be3612cce9e20f84eeec2ff98b36f8f2ff9ecf Mon Sep 17 00:00:00 2001 From: Evgenii Mukhamadiarov Date: Tue, 29 Oct 2019 23:16:47 +0300 Subject: [PATCH 194/194] clean-code --- tes-adapter/src/main/resources/messages.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tes-adapter/src/main/resources/messages.properties b/tes-adapter/src/main/resources/messages.properties index 424d383b10..2976788d00 100644 --- a/tes-adapter/src/main/resources/messages.properties +++ b/tes-adapter/src/main/resources/messages.properties @@ -2,7 +2,7 @@ error.null.param=Error: the required parameter ''{0}'' is null or empty error.parameter.required=Parameter ''{0}'' id required for ''{1}'' podId. error.parameter.non.scalar.type=Unable to resolve parameter value ''{0}''. Reference value for field ''{1}'' is used as a parameter value. -error.parameter.incompatible.content=Parameter (id ''{0}'') with undesirable content.uri=/v1/tasks/%20:cancel;client=127.0.0.1 +error.parameter.incompatible.content=Parameter (id ''{0}'') with undesirable content. debug.token.found.in.request=Auth token found in HEADERS or COOKIE for request ''{0}'' debug.ip.is.accepted=Get default auth token, since request host is from accepted ip range for request ''{0}''