diff --git a/extraction-api-demo/src/main/java/cern/nxcals/api/metadata/winccoa/HandleNXCALS7964.java b/extraction-api-demo/src/main/java/cern/nxcals/api/metadata/winccoa/HandleNXCALS7964.java
new file mode 100644
index 0000000000000000000000000000000000000000..fb806bc24de18912f57161eb79c28f8903a1e4e8
--- /dev/null
+++ b/extraction-api-demo/src/main/java/cern/nxcals/api/metadata/winccoa/HandleNXCALS7964.java
@@ -0,0 +1,272 @@
+package cern.nxcals.api.metadata.winccoa;
+
+import cern.nxcals.api.config.SparkContext;
+import cern.nxcals.api.domain.Entity;
+import cern.nxcals.api.domain.SystemSpec;
+import cern.nxcals.api.domain.TimeWindow;
+import cern.nxcals.api.domain.Variable;
+import cern.nxcals.api.domain.VariableConfig;
+import cern.nxcals.api.extraction.metadata.EntityService;
+import cern.nxcals.api.extraction.metadata.ServiceClientFactory;
+import cern.nxcals.api.extraction.metadata.VariableService;
+import cern.nxcals.api.extraction.metadata.queries.Entities;
+import cern.nxcals.api.extraction.metadata.queries.Variables;
+import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.annotation.Import;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+@Import(SparkContext.class)
+@SpringBootApplication
+@Slf4j
+public class HandleNXCALS7964 {
+ static {
+ String user = System.getProperty("user.name");
+ // System.setProperty("kerberos.principal", "acclog");
+ // System.setProperty("kerberos.keytab", "/opt/" + user + "/.keytab-acclog");
+ System.setProperty("kerberos.principal", user);
+ System.setProperty("kerberos.keytab", "/opt/" + user + "/.keytab");
+ System.setProperty("service.url",
+ "https://cs-ccr-nxcals6.cern.ch:19093,https://cs-ccr-nxcals7.cern.ch:19093,https://cs-ccr-nxcals8.cern.ch:19093");
+ // System.setProperty("service.url",
+ // "https://nxcals-rmucha-2.cern.ch:19093");
+ }
+
+ private static final String system = "WINCCOA";
+ private static final String winccoaEntityKey = "variable_name";
+
+ private static final String[] variableList = new String[] {
+ "QRWPU_LSS6_1TT203P.COMMAND",
+ "ACFGA_LSS6_TT851.COMMAND",
+ "QUIS_TA6_TT710.COMMAND",
+ "QRIKS_LSS6_TT732.COMMAND",
+ "ACFGA_LSS6_TT825.COMMAND",
+ "QRIKC_LSS6_PT870.COMMAND",
+ "QUIS_TA6_TT796.COMMAND",
+ "ACFGA_LSS6_TT805.COMMAND",
+ "ACFGA_LSS6_TT852.COMMAND",
+ "QRIKC_LSS6_TT911.COMMAND",
+ "QUIS_TA6_TT731.COMMAND",
+ "ACFGA_LSS6_LT822.COMMAND",
+ "QRIKS_LSS6_TT798.COMMAND",
+ "ACFGA_LSS6_EH852.CURVEID",
+ "QRWPU_LSS6_2PT212.COMMAND",
+ "ACFGA_LSS6_EH862.CURVEID",
+ "QRWPU_LSS6_TT200.COMMAND",
+ "ACFGA_LSS6_TT802.COMMAND",
+ "QRIKC_LSS6_TT981.COMMAND",
+ "QRIKC_LSS6_PT977.COMMAND",
+ "ACFGA_LSS6_EH851.CURVEID",
+ "QRIKS_LSS6_TT788.COMMAND",
+ "QUIN_TA6_PDT454.COMMAND",
+ "ACFGA_LSS6_TT804.COMMAND",
+ "QUIN_TA6_EH454.CURVEID",
+ "QRIKC_LSS6_PT901.COMMAND",
+ "QRIKS_LSS6_EH732.CURVEID",
+ "QRIKC_LSS6_PT386.COMMAND",
+ "QRWPU_LSS6_PT501.COMMAND",
+ "QRWPU_LSS6_PT200.COMMAND",
+ "ACFGA_LSS6_TT812.COMMAND",
+ "QUIN_TA6_TT452.COMMAND",
+ "ACFGA_LSS6_TT819.COMMAND",
+ "ACFGA_LSS6_TT823.COMMAND",
+ "ACFGA_LSS6_TT831.COMMAND",
+ "ACFGA_LSS6_TT803.COMMAND",
+ "QUIS_TA6_TT780.COMMAND",
+ "ACFGA_LSS6_TT821.COMMAND",
+ "ACFGA_LSS6_TT817.COMMAND",
+ "QRWPU_LSS6_2TT203P.COMMAND",
+ "QUIN_TA6_TT454.COMMAND",
+ "QUIS_TA6_PT731.COMMAND",
+ "QRWPU_LSS6_1PT203.COMMAND",
+ "ACFGA_LSS6_TT820.COMMAND",
+ "QRHCA_LSS6_TT979.COMMAND",
+ "QUIN_TA6_PT453.COMMAND",
+ "ACFGA_LSS6_TT818.COMMAND",
+ "ACFGA_LSS6_TT870P.COMMAND",
+ "QRWPU_LSS6_2PT203.COMMAND",
+ "QRWPU_LSS6_PT201.COMMAND",
+ "ACFGA_LSS6_TT822.COMMAND",
+ "QUIN_TA6_TT459.COMMAND",
+ "QUIN_TA6_EH453.CURVEID",
+ "QRIKC_LSS6_PT976.COMMAND",
+ "ACFGA_LSS6_TT816.COMMAND",
+ "ACFGA_LSS6_TT810.COMMAND",
+ "QRHCA_LSS6_TT540.COMMAND",
+ "QUIS_TA6_EH731.CURVEID",
+ "QRHCA_LSS6_TT541.COMMAND",
+ "ACFGA_LSS6_TT814.COMMAND",
+ "ACFGA_LSS6_TT815.COMMAND",
+ "QRLKS_LSS6_TT711.COMMAND",
+ "ACFGA_LSS6_TT841.COMMAND",
+ "QRWPU_TA6_PT235.COMMAND",
+ "QUIN_TA6_TT454P.COMMAND",
+ "QUIN_TA6_TT458.COMMAND",
+ "QRIKS_LSS6_TT782.COMMAND",
+ "ACFGA_LSS6_EH863.CURVEID",
+ "ACFGA_LSS6_TT832.COMMAND",
+ "QRWPU_LSS6_2PT202.COMMAND",
+ "ACFGA_LSS6_EH821.CURVEID",
+ "QRIKC_LSS6_PT981.COMMAND",
+ "QRWPU_LSS6_1TT211.COMMAND",
+ "ACFGA_LSS6_TT801.COMMAND",
+ "QRIKC_LSS6_PT871.COMMAND",
+ "QRIKC_LSS6_TT901.COMMAND",
+ "QUIS_TA6_LT731.COMMAND",
+ "QRLKS_LSS6_TT787.COMMAND",
+ "ACFGA_LSS6_EH861.CURVEID",
+ "CBWMB_BA6_QAICC.COMMAND",
+ "QRIKS_LSS6_TT789.COMMAND",
+ "QRLKS_LSS6_TT797.COMMAND",
+ "QRWPU_LSS6_2TT211.COMMAND",
+ "ACFGA_LSS6_TT824.COMMAND",
+ "QRIKS_LSS6_TT712.COMMAND",
+ "QRIKS_LSS6_TT783.COMMAND",
+ "ACFGA_LSS6_TT840.COMMAND",
+ "ACFGA_LSS6_TT811.COMMAND",
+ "QRIKC_LSS6_TT971.COMMAND",
+ "QRLKS_LSS6_TT781.COMMAND",
+ "QRWPU_LSS6_1PT212.COMMAND",
+ "QRWPU_LSS6_1PT202.COMMAND",
+ "ACFGA_LSS6_TT813.COMMAND",
+ "QRWPU_LSS6_TT201P.COMMAND",
+ "QRIKC_LSS6_TT910.COMMAND",
+ "QUIS_TA6_TT786.COMMAND",
+ "ACFGA_LSS6_LT821.COMMAND",
+ "ACFGA_LSS6_EH822.CURVEID",
+ "QUIS_TA6_TT706.COMMAND",
+ "QRIKS_LSS6_PT732.COMMAND",
+ "QRIKS_LSS6_PT788.COMMAND",
+ "QRIKS_LSS6_LT732.COMMAND"
+ };
+
+ private void generateData() {
+ // generate data - for test in dev env
+ // EntityService entityService = ServiceClientFactory.createEntityService();
+ // VariableService variableService = ServiceClientFactory.createVariableService();
+ // SystemSpec winccoaSystemSpec = ServiceClientFactory.createSystemSpecService().findByName(system).orElseThrow();
+ // for (String variableName : variableList) {
+ // Entity entity = entityService.createEntity(winccoaSystemSpec.getId(),
+ // Map.of(winccoaEntityKey, variableName), Map.of("application_arcgroup", "CIET_SPSCRB.EVENTHISTORY"));
+ // // createdEntities.put(variableName, entity);
+ // variableService.create(Variable.builder()
+ // .systemSpec(winccoaSystemSpec)
+ // .variableName(variableName)
+ // .configs(ImmutableSortedSet.of(
+ // VariableConfig.builder()
+ // .validity(TimeWindow.infinite())
+ // .entityId(entity.getId())
+ // .fieldName("value")
+ // .build()
+ // ))
+ // .build());
+ // }
+ }
+
+ public static void main(String[] args) throws IOException {
+ ConfigurableApplicationContext context = SpringApplication.run(HandleNXCALS7964.class, args);
+ EntityService entityService = ServiceClientFactory.createEntityService();
+ VariableService variableService = ServiceClientFactory.createVariableService();
+ SystemSpec winccoaSystemSpec = ServiceClientFactory.createSystemSpecService().findByName(system).orElseThrow();
+
+ // rename entities
+ Map<String, Object>[] keyValues = Arrays.stream(variableList).map(
+ variableName -> Map.of("variable_name", variableName)
+ ).toArray(Map[]::new);
+ Set<Entity> entitiesToRename = entityService.findAll(
+ Entities.suchThat().systemName().eq("WINCCOA").and().keyValues().in(winccoaSystemSpec, keyValues));
+
+ if (entitiesToRename.size() != variableList.length) {
+ Set<String> foundEntities = entitiesToRename.stream().map(Entity::getEntityKeyValues).map(x -> x.toString())
+ .collect(Collectors.toSet());
+ Set<String> missingVariables = Sets.difference(new HashSet<>(List.of(variableList)), foundEntities);
+ throw new IllegalArgumentException(
+ "Incorrect variable list, following entities were not found: " + missingVariables);
+ }
+
+ System.out.println(entitiesToRename.size());
+
+ Set<Entity> modifiedEntities = new HashSet<>();
+ for (Entity entity : entitiesToRename) {
+ Map<String, Object> keyValue = entity.getEntityKeyValues();
+ Entity modifiedEntity = entity.toBuilder()
+ .entityKeyValues(Map.of(winccoaEntityKey, keyValue.get(winccoaEntityKey).toString() + "_old"))
+ .build();
+ System.out.println(modifiedEntity.getEntityKeyValues());
+ modifiedEntities.add(modifiedEntity);
+ }
+
+ entityService.updateEntities(modifiedEntities);
+
+ // create a new entities
+ Map<String, Entity> createdEntities = new HashMap<>();
+ for (String variableName : variableList) {
+ Entity entity = entityService.createEntity(winccoaSystemSpec.getId(),
+ Map.of(winccoaEntityKey, variableName), Map.of("application_arcgroup", "SPSCRABCIET.EVENTHISTORY"));
+ createdEntities.put(variableName, entity);
+ }
+
+ //
+ // find created entities - in case, if variable update fail
+ // Map<String, Entity> createdEntities = new HashMap<>();
+ // for (String variableName : variableList) {
+ // Entity entity = entityService.findOne(Entities.suchThat().systemName().eq(system).and().keyValues()
+ // .eq(winccoaSystemSpec, Map.of(winccoaEntityKey, variableName))).orElseThrow();
+ // createdEntities.put(variableName, entity);
+ // }
+
+ // update variables
+ Set<Variable> variables = variableService.findAll(Variables.suchThat().variableName().in(variableList));
+
+ if (variables.size() != variableList.length) {
+ Set<String> foundVariableNames = variables.stream().map(Variable::getVariableName)
+ .collect(Collectors.toSet());
+ Set<String> missingVariables = Sets.difference(new HashSet<>(List.of(variableList)), foundVariableNames);
+ throw new IllegalArgumentException(
+ "Incorrect variable list, following variables were not found: " + missingVariables);
+ }
+
+ Set<Variable> modifiedVariables = new HashSet<>();
+ Instant now = Instant.now();
+ for (Variable variable : variables) {
+ SortedSet<VariableConfig> configs = new TreeSet<>(variable.getConfigs());
+ VariableConfig closedConfig = null;
+ for (VariableConfig config : new HashSet<>(configs)) {
+ TimeWindow validity = config.getValidity();
+ if (validity.isRightInfinite()) {
+ configs.remove(config);
+ configs.add(config.toBuilder().validity(validity.rightLimit(now)).build());
+ closedConfig = config;
+ }
+ }
+ if (closedConfig == null) {
+ throw new RuntimeException("No variable config was closed!");
+ }
+ configs.add(
+ VariableConfig.builder()
+ .entityId(createdEntities.get(variable.getVariableName()).getId())
+ .fieldName(closedConfig.getFieldName())
+ .validity(TimeWindow.after(now))
+ .build()
+ );
+ modifiedVariables.add(variable.toBuilder().configs(configs).build());
+ }
+
+ variableService.updateAll(modifiedVariables);
+ System.out.println("Done!");
+ }
+}
diff --git a/extraction-api-demo/src/main/resources/application.yml b/extraction-api-demo/src/main/resources/application.yml
index 171f9b62f17116eeca3485446a4c43c0e5755aa5..215f0416996345e33c9789b7ac22502bf3cce59f 100644
--- a/extraction-api-demo/src/main/resources/application.yml
+++ b/extraction-api-demo/src/main/resources/application.yml
@@ -55,8 +55,8 @@ spring.profiles.active: prod
---
spring:
profiles: prod
-spark.masterType: yarn
-#spark.masterType: local[*]
+#spark.masterType: yarn
+spark.masterType: local[*]
spark.properties:
spark.submit.deployMode: client