Skip to content
Snippets Groups Projects

NXCALS-7569 Improve generation of tmp working dir for compactor jobs

Merged Rafal Mucha requested to merge NXCALS-7569-fix-generating-tmp-working-dir into develop
1 file
+ 4
3
Compare changes
  • Side-by-side
  • Inline
@@ -37,6 +37,7 @@ import java.util.List;
@@ -37,6 +37,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map;
import java.util.Objects;
import java.util.Objects;
import java.util.Set;
import java.util.Set;
 
import java.util.UUID;
import java.util.function.UnaryOperator;
import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
import java.util.stream.Collectors;
@@ -78,8 +79,8 @@ public abstract class BaseJobProcessor<T extends DataProcessingJob> implements J
@@ -78,8 +79,8 @@ public abstract class BaseJobProcessor<T extends DataProcessingJob> implements J
private final LoadingCache<Long, String> timestampFieldsCache;
private final LoadingCache<Long, String> timestampFieldsCache;
protected BaseJobProcessor(@NonNull FileSystem fileSystem, @NonNull Lazy<SparkSession> ctx,
protected BaseJobProcessor(@NonNull FileSystem fileSystem, @NonNull Lazy<SparkSession> ctx,
@NonNull InternalSystemSpecService systemService,
@NonNull InternalSystemSpecService systemService,
@NonNull @Value("${compactor.work.dir}") String compactionDir) {
@NonNull @Value("${compactor.work.dir}") String compactionDir) {
this.fs = fileSystem;
this.fs = fileSystem;
this.sparkSessionLazy = ctx;
this.sparkSessionLazy = ctx;
this.systemService = systemService;
this.systemService = systemService;
@@ -199,7 +200,7 @@ public abstract class BaseJobProcessor<T extends DataProcessingJob> implements J
@@ -199,7 +200,7 @@ public abstract class BaseJobProcessor<T extends DataProcessingJob> implements J
protected Path createTempCompactionPath(T job, String filePrefix) {
protected Path createTempCompactionPath(T job, String filePrefix) {
return new Path(compactionDir + Path.SEPARATOR + "compaction" + job.getDestinationDir().getPath()
return new Path(compactionDir + Path.SEPARATOR + "compaction" + job.getDestinationDir().getPath()
.replace(Path.SEPARATOR_CHAR, '-') + "-" + filePrefix + System.currentTimeMillis());
.replace(Path.SEPARATOR_CHAR, '-') + "-" + filePrefix + UUID.randomUUID());
}
}
private void startLog(T job) {
private void startLog(T job) {
Loading