Skip to content
Snippets Groups Projects
Commit c641d689 authored by Rafal Mucha's avatar Rafal Mucha
Browse files

Merge branch 'NXCALS-7964-fix-winccoa-metadata' into 'develop' 

NXCALS-7964 Add utility script to handle changes in metadata for WINCCOA

Closes NXCALS-7964

See merge request !2088

Changelog: added
parents fa8dfcc3 c466df01
No related branches found
No related tags found
1 merge request!2088NXCALS-7964 Add utility script to handle changes in metadata for WINCCOA
Hide whitespace changes
Inline Side-by-side
extraction-api-demo/src/main/java/cern/nxcals/api/metadata/winccoa/HandleNXCALS7964.java 0 → 100644
+ 272
− 0
View file @ c641d689
package cern.nxcals.api.metadata.winccoa;
import cern.nxcals.api.config.SparkContext;
import cern.nxcals.api.domain.Entity;
import cern.nxcals.api.domain.SystemSpec;
import cern.nxcals.api.domain.TimeWindow;
import cern.nxcals.api.domain.Variable;
import cern.nxcals.api.domain.VariableConfig;
import cern.nxcals.api.extraction.metadata.EntityService;
import cern.nxcals.api.extraction.metadata.ServiceClientFactory;
import cern.nxcals.api.extraction.metadata.VariableService;
import cern.nxcals.api.extraction.metadata.queries.Entities;
import cern.nxcals.api.extraction.metadata.queries.Variables;
import com.google.common.collect.Sets;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.Import;
import java.io.IOException;
import java.time.Instant;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.stream.Collectors;
@Import(SparkContext.class)
@SpringBootApplication
@Slf4j
public class HandleNXCALS7964 {
static {
String user = System.getProperty("user.name");
// System.setProperty("kerberos.principal", "acclog");
// System.setProperty("kerberos.keytab", "/opt/" + user + "/.keytab-acclog");
System.setProperty("kerberos.principal", user);
System.setProperty("kerberos.keytab", "/opt/" + user + "/.keytab");
System.setProperty("service.url",
"https://cs-ccr-nxcals6.cern.ch:19093,https://cs-ccr-nxcals7.cern.ch:19093,https://cs-ccr-nxcals8.cern.ch:19093");
// System.setProperty("service.url",
// "https://nxcals-rmucha-2.cern.ch:19093");
}
private static final String system = "WINCCOA";
private static final String winccoaEntityKey = "variable_name";
private static final String[] variableList = new String[] {
"QRWPU_LSS6_1TT203P.COMMAND",
"ACFGA_LSS6_TT851.COMMAND",
"QUIS_TA6_TT710.COMMAND",
"QRIKS_LSS6_TT732.COMMAND",
"ACFGA_LSS6_TT825.COMMAND",
"QRIKC_LSS6_PT870.COMMAND",
"QUIS_TA6_TT796.COMMAND",
"ACFGA_LSS6_TT805.COMMAND",
"ACFGA_LSS6_TT852.COMMAND",
"QRIKC_LSS6_TT911.COMMAND",
"QUIS_TA6_TT731.COMMAND",
"ACFGA_LSS6_LT822.COMMAND",
"QRIKS_LSS6_TT798.COMMAND",
"ACFGA_LSS6_EH852.CURVEID",
"QRWPU_LSS6_2PT212.COMMAND",
"ACFGA_LSS6_EH862.CURVEID",
"QRWPU_LSS6_TT200.COMMAND",
"ACFGA_LSS6_TT802.COMMAND",
"QRIKC_LSS6_TT981.COMMAND",
"QRIKC_LSS6_PT977.COMMAND",
"ACFGA_LSS6_EH851.CURVEID",
"QRIKS_LSS6_TT788.COMMAND",
"QUIN_TA6_PDT454.COMMAND",
"ACFGA_LSS6_TT804.COMMAND",
"QUIN_TA6_EH454.CURVEID",
"QRIKC_LSS6_PT901.COMMAND",
"QRIKS_LSS6_EH732.CURVEID",
"QRIKC_LSS6_PT386.COMMAND",
"QRWPU_LSS6_PT501.COMMAND",
"QRWPU_LSS6_PT200.COMMAND",
"ACFGA_LSS6_TT812.COMMAND",
"QUIN_TA6_TT452.COMMAND",
"ACFGA_LSS6_TT819.COMMAND",
"ACFGA_LSS6_TT823.COMMAND",
"ACFGA_LSS6_TT831.COMMAND",
"ACFGA_LSS6_TT803.COMMAND",
"QUIS_TA6_TT780.COMMAND",
"ACFGA_LSS6_TT821.COMMAND",
"ACFGA_LSS6_TT817.COMMAND",
"QRWPU_LSS6_2TT203P.COMMAND",
"QUIN_TA6_TT454.COMMAND",
"QUIS_TA6_PT731.COMMAND",
"QRWPU_LSS6_1PT203.COMMAND",
"ACFGA_LSS6_TT820.COMMAND",
"QRHCA_LSS6_TT979.COMMAND",
"QUIN_TA6_PT453.COMMAND",
"ACFGA_LSS6_TT818.COMMAND",
"ACFGA_LSS6_TT870P.COMMAND",
"QRWPU_LSS6_2PT203.COMMAND",
"QRWPU_LSS6_PT201.COMMAND",
"ACFGA_LSS6_TT822.COMMAND",
"QUIN_TA6_TT459.COMMAND",
"QUIN_TA6_EH453.CURVEID",
"QRIKC_LSS6_PT976.COMMAND",
"ACFGA_LSS6_TT816.COMMAND",
"ACFGA_LSS6_TT810.COMMAND",
"QRHCA_LSS6_TT540.COMMAND",
"QUIS_TA6_EH731.CURVEID",
"QRHCA_LSS6_TT541.COMMAND",
"ACFGA_LSS6_TT814.COMMAND",
"ACFGA_LSS6_TT815.COMMAND",
"QRLKS_LSS6_TT711.COMMAND",
"ACFGA_LSS6_TT841.COMMAND",
"QRWPU_TA6_PT235.COMMAND",
"QUIN_TA6_TT454P.COMMAND",
"QUIN_TA6_TT458.COMMAND",
"QRIKS_LSS6_TT782.COMMAND",
"ACFGA_LSS6_EH863.CURVEID",
"ACFGA_LSS6_TT832.COMMAND",
"QRWPU_LSS6_2PT202.COMMAND",
"ACFGA_LSS6_EH821.CURVEID",
"QRIKC_LSS6_PT981.COMMAND",
"QRWPU_LSS6_1TT211.COMMAND",
"ACFGA_LSS6_TT801.COMMAND",
"QRIKC_LSS6_PT871.COMMAND",
"QRIKC_LSS6_TT901.COMMAND",
"QUIS_TA6_LT731.COMMAND",
"QRLKS_LSS6_TT787.COMMAND",
"ACFGA_LSS6_EH861.CURVEID",
"CBWMB_BA6_QAICC.COMMAND",
"QRIKS_LSS6_TT789.COMMAND",
"QRLKS_LSS6_TT797.COMMAND",
"QRWPU_LSS6_2TT211.COMMAND",
"ACFGA_LSS6_TT824.COMMAND",
"QRIKS_LSS6_TT712.COMMAND",
"QRIKS_LSS6_TT783.COMMAND",
"ACFGA_LSS6_TT840.COMMAND",
"ACFGA_LSS6_TT811.COMMAND",
"QRIKC_LSS6_TT971.COMMAND",
"QRLKS_LSS6_TT781.COMMAND",
"QRWPU_LSS6_1PT212.COMMAND",
"QRWPU_LSS6_1PT202.COMMAND",
"ACFGA_LSS6_TT813.COMMAND",
"QRWPU_LSS6_TT201P.COMMAND",
"QRIKC_LSS6_TT910.COMMAND",
"QUIS_TA6_TT786.COMMAND",
"ACFGA_LSS6_LT821.COMMAND",
"ACFGA_LSS6_EH822.CURVEID",
"QUIS_TA6_TT706.COMMAND",
"QRIKS_LSS6_PT732.COMMAND",
"QRIKS_LSS6_PT788.COMMAND",
"QRIKS_LSS6_LT732.COMMAND"
};
private void generateData() {
// generate data - for test in dev env
// EntityService entityService = ServiceClientFactory.createEntityService();
// VariableService variableService = ServiceClientFactory.createVariableService();
// SystemSpec winccoaSystemSpec = ServiceClientFactory.createSystemSpecService().findByName(system).orElseThrow();
// for (String variableName : variableList) {
// Entity entity = entityService.createEntity(winccoaSystemSpec.getId(),
// Map.of(winccoaEntityKey, variableName), Map.of("application_arcgroup", "CIET_SPSCRB.EVENTHISTORY"));
// // createdEntities.put(variableName, entity);
// variableService.create(Variable.builder()
// .systemSpec(winccoaSystemSpec)
// .variableName(variableName)
// .configs(ImmutableSortedSet.of(
// VariableConfig.builder()
// .validity(TimeWindow.infinite())
// .entityId(entity.getId())
// .fieldName("value")
// .build()
// ))
// .build());
// }
}
public static void main(String[] args) throws IOException {
ConfigurableApplicationContext context = SpringApplication.run(HandleNXCALS7964.class, args);
EntityService entityService = ServiceClientFactory.createEntityService();
VariableService variableService = ServiceClientFactory.createVariableService();
SystemSpec winccoaSystemSpec = ServiceClientFactory.createSystemSpecService().findByName(system).orElseThrow();
// rename entities
Map<String, Object>[] keyValues = Arrays.stream(variableList).map(
variableName -> Map.of("variable_name", variableName)
).toArray(Map[]::new);
Set<Entity> entitiesToRename = entityService.findAll(
Entities.suchThat().systemName().eq("WINCCOA").and().keyValues().in(winccoaSystemSpec, keyValues));
if (entitiesToRename.size() != variableList.length) {
Set<String> foundEntities = entitiesToRename.stream().map(Entity::getEntityKeyValues).map(x -> x.toString())
.collect(Collectors.toSet());
Set<String> missingVariables = Sets.difference(new HashSet<>(List.of(variableList)), foundEntities);
throw new IllegalArgumentException(
"Incorrect variable list, following entities were not found: " + missingVariables);
}
System.out.println(entitiesToRename.size());
Set<Entity> modifiedEntities = new HashSet<>();
for (Entity entity : entitiesToRename) {
Map<String, Object> keyValue = entity.getEntityKeyValues();
Entity modifiedEntity = entity.toBuilder()
.entityKeyValues(Map.of(winccoaEntityKey, keyValue.get(winccoaEntityKey).toString() + "_old"))
.build();
System.out.println(modifiedEntity.getEntityKeyValues());
modifiedEntities.add(modifiedEntity);
}
entityService.updateEntities(modifiedEntities);
// create a new entities
Map<String, Entity> createdEntities = new HashMap<>();
for (String variableName : variableList) {
Entity entity = entityService.createEntity(winccoaSystemSpec.getId(),
Map.of(winccoaEntityKey, variableName), Map.of("application_arcgroup", "SPSCRABCIET.EVENTHISTORY"));
createdEntities.put(variableName, entity);
}
//
// find created entities - in case, if variable update fail
// Map<String, Entity> createdEntities = new HashMap<>();
// for (String variableName : variableList) {
// Entity entity = entityService.findOne(Entities.suchThat().systemName().eq(system).and().keyValues()
// .eq(winccoaSystemSpec, Map.of(winccoaEntityKey, variableName))).orElseThrow();
// createdEntities.put(variableName, entity);
// }
// update variables
Set<Variable> variables = variableService.findAll(Variables.suchThat().variableName().in(variableList));
if (variables.size() != variableList.length) {
Set<String> foundVariableNames = variables.stream().map(Variable::getVariableName)
.collect(Collectors.toSet());
Set<String> missingVariables = Sets.difference(new HashSet<>(List.of(variableList)), foundVariableNames);
throw new IllegalArgumentException(
"Incorrect variable list, following variables were not found: " + missingVariables);
}
Set<Variable> modifiedVariables = new HashSet<>();
Instant now = Instant.now();
for (Variable variable : variables) {
SortedSet<VariableConfig> configs = new TreeSet<>(variable.getConfigs());
VariableConfig closedConfig = null;
for (VariableConfig config : new HashSet<>(configs)) {
TimeWindow validity = config.getValidity();
if (validity.isRightInfinite()) {
configs.remove(config);
configs.add(config.toBuilder().validity(validity.rightLimit(now)).build());
closedConfig = config;
}
}
if (closedConfig == null) {
throw new RuntimeException("No variable config was closed!");
}
configs.add(
VariableConfig.builder()
.entityId(createdEntities.get(variable.getVariableName()).getId())
.fieldName(closedConfig.getFieldName())
.validity(TimeWindow.after(now))
.build()
);
modifiedVariables.add(variable.toBuilder().configs(configs).build());
}
variableService.updateAll(modifiedVariables);
System.out.println("Done!");
}
}
extraction-api-demo/src/main/resources/application.yml
+ 2
− 2
View file @ c641d689
...@@ -55,8 +55,8 @@ spring.profiles.active: prod ...@@ -55,8 +55,8 @@ spring.profiles.active: prod
--- ---
spring: spring:
profiles: prod profiles: prod
spark.masterType: yarn #spark.masterType: yarn
#spark.masterType: local[*] spark.masterType: local[*]
spark.properties: spark.properties:
spark.submit.deployMode: client spark.submit.deployMode: client
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment