Add debugDir parameter to dynamic-simulation, dynamic-security-analysis, shortcircuit and ampl-executor (#3455)

* Add debugDir parameter to dynmaic-simulation run() method
* Move debugDir to DynamicSimulationParameters
* copy debug files to debugDir/workingDir + use getFileSystem()
* fix unit test dynamic simulation parameter
* ShortcircuitParameters 1.4
* add debugDir for DynamicSecurityAnalysisParameters
* Manage version for DynamicSimulationParameters
* increase version for DynamicSecurityAnalysisParameters
* manage retro compatibility in DynamicSimulationParametersDeserializer
* Add debugDir to AmplParameters
* fix DynamicSimulationParameters constr
* Rename debugDir to dumpDir in ExecutionEnvironment
* Add documentation
* do not serialize debugDir if null

Signed-off-by: Etienne Homer <[email protected]>
Co-authored-by: Thang PHAM <[email protected]>

Author: etiennehomer

ampl-executor/src/main/java/com/powsybl/ampl/executor/AmplModelRunner.java

@@ -29,7 +29,7 @@ public static AmplResults run(Network network, String variantId, AmplModel model
 
     public static CompletableFuture<AmplResults> runAsync(Network network, String variantId, AmplModel model, ComputationManager manager,
                                                           AmplParameters parameters) {
-        ExecutionEnvironment env = new ExecutionEnvironment(Collections.emptyMap(), "ampl_", parameters.isDebug());
+        ExecutionEnvironment env = new ExecutionEnvironment(Collections.emptyMap(), "ampl_", parameters.isDebug(), parameters.getDebugDir());
         AmplModelExecutionHandler handler = new AmplModelExecutionHandler(model, network, variantId, AmplConfig.load(),
                 parameters);
         return manager.execute(env, handler);

ampl-executor/src/main/java/com/powsybl/ampl/executor/AmplParameters.java

@@ -36,6 +36,13 @@ public interface AmplParameters {
      */
     boolean isDebug();
 
+    /**
+     * Get the directory where execution files will be dumped
+     *
+     * @return the debug directory
+     */
+    String getDebugDir();
+
     /**
      * Configuration for AmplExporter
      */

ampl-executor/src/main/java/com/powsybl/ampl/executor/EmptyAmplParameters.java

@@ -32,6 +32,11 @@ public boolean isDebug() {
         return false;
     }
 
+    @Override
+    public String getDebugDir() {
+        return null;
+    }
+
     @Override
     public AmplExportConfig getAmplExportConfig() {
         return null;

ampl-executor/src/test/java/com/powsybl/ampl/executor/SimpleAmplParameters.java

@@ -73,6 +73,11 @@ public boolean isDebug() {
         return true;
     }
 
+    @Override
+    public String getDebugDir() {
+        return null;
+    }
+
     @Override
     public AmplExportConfig getAmplExportConfig() {
         return new AmplExportConfig(AmplExportConfig.ExportScope.ALL, false, AmplExportConfig.ExportActionType.CURATIVE, false, false, AmplExportVersion.defaultVersion(), false);

computation-local/src/main/java/com/powsybl/computation/local/LocalComputationManager.java

@@ -12,6 +12,7 @@
 import com.google.common.io.ByteStreams;
 import com.powsybl.commons.PowsyblException;
 import com.powsybl.commons.config.PlatformConfig;
+import com.powsybl.commons.io.FileUtil;
 import com.powsybl.commons.io.WorkingDirectory;
 import com.powsybl.computation.*;
 import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
@@ -20,10 +21,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.UncheckedIOException;
+import java.io.*;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
@@ -146,7 +144,7 @@ private interface ExecutionMonitor {
 
     }
 
-    private ExecutionReport execute(Path workingDir, List<CommandExecution> commandExecutionList, Map<String, String> variables, ComputationParameters computationParameters, ExecutionMonitor monitor)
+    private ExecutionReport execute(Path workingDir, Path dumpDir, List<CommandExecution> commandExecutionList, Map<String, String> variables, ComputationParameters computationParameters, ExecutionMonitor monitor)
             throws InterruptedException {
         // TODO concurrent
         List<ExecutionError> errors = new ArrayList<>();
@@ -155,7 +153,7 @@ private ExecutionReport execute(Path workingDir, List<CommandExecution> commandE
         for (CommandExecution commandExecution : commandExecutionList) {
             Command command = commandExecution.getCommand();
             CountDownLatch latch = new CountDownLatch(commandExecution.getExecutionCount());
-            ExecutionParameters executionParameters = new ExecutionParameters(workingDir, commandExecution, variables, computationParameters, executionSubmitter,
+            ExecutionParameters executionParameters = new ExecutionParameters(workingDir, dumpDir, commandExecution, variables, computationParameters, executionSubmitter,
                 command, latch, errors, monitor);
             IntStream.range(0, commandExecution.getExecutionCount()).forEach(idx -> performSingleExecution(executionParameters, idx));
             latch.await();
@@ -173,7 +171,7 @@ private ExecutionReport execute(Path workingDir, List<CommandExecution> commandE
         return new DefaultExecutionReport(workingDir, errors);
     }
 
-    private record ExecutionParameters(Path workingDir, CommandExecution commandExecution,
+    private record ExecutionParameters(Path workingDir, Path dumpDir, CommandExecution commandExecution,
                                        Map<String, String> variables, ComputationParameters computationParameters,
                                        ExecutorService executionSubmitter, Command command, CountDownLatch latch,
                                        List<ExecutionError> errors, ExecutionMonitor monitor) {
@@ -207,6 +205,17 @@ private void performSingleExecution(ExecutionParameters executionParameters, int
             } catch (Exception e) {
                 LOGGER.warn(e.getMessage(), e);
             } finally {
+                if (executionParameters.dumpDir != null) {
+                    try {
+                        Path sourcePath = executionParameters.workingDir;
+                        Path destinationPath = executionParameters.dumpDir.resolve(executionParameters.workingDir.getFileName());
+                        FileUtil.createDirectory(destinationPath);
+                        FileUtil.copyDir(sourcePath, destinationPath);
+
+                    } catch (IOException e) {
+                        LOGGER.warn(e.getMessage(), e);
+                    }
+                }
                 executionParameters.latch.countDown();
                 exit();
             }
@@ -363,8 +372,14 @@ private <R> R doExecute(ExecutionEnvironment environment, ExecutionHandler<R> ha
             List<CommandExecution> commandExecutionList = handler.before(workingDir.toPath());
 
             ExecutionReport report;
+
             try {
-                report = execute(workingDir.toPath(), commandExecutionList, environment.getVariables(), parameters, handler::onExecutionCompletion);
+                report = execute(workingDir.toPath(),
+                        environment.getDumpDir() != null ? config.getLocalDir().getFileSystem().getPath(environment.getDumpDir()) : null,
+                        commandExecutionList,
+                        environment.getVariables(),
+                        parameters,
+                        handler::onExecutionCompletion);
             } catch (InterruptedException exc) {
                 localCommandExecutor.stop(workingDir.toPath());
                 throw exc;

computation-local/src/test/java/com/powsybl/computation/local/LocalComputationManagerTest.java

@@ -46,6 +46,8 @@ class LocalComputationManagerTest {
 
     private Path localDir;
 
+    private static final String DEBUG_DIR = "/tmp/debugDir";
+
     private LocalComputationConfig config;
 
     @BeforeEach
@@ -98,7 +100,7 @@ public int execute(String program, List<String> args, Path outFile, Path errFile
             }
         };
         try (ComputationManager computationManager = new LocalComputationManager(config, localCommandExecutor, ForkJoinPool.commonPool())) {
-            computationManager.execute(new ExecutionEnvironment(ImmutableMap.of("var1", "val1"), PREFIX, false),
+            computationManager.execute(new ExecutionEnvironment(ImmutableMap.of("var1", "val1"), PREFIX, false, DEBUG_DIR),
                     new AbstractExecutionHandler<Object>() {
                         @Override
                         public List<CommandExecution> before(Path workingDir) throws IOException {
@@ -131,7 +133,7 @@ public Object after(Path workingDir, ExecutionReport report) throws IOException
                             assertEquals("prog1_cmd", report.getErrors().get(0).getCommand().getId());
                             assertEquals(0, report.getErrors().get(0).getIndex());
                             assertEquals(1, report.getErrors().get(0).getExitCode());
-
+                            assertTrue(Files.exists(fileSystem.getPath(DEBUG_DIR).resolve(workingDir)));
                             return null;
                         }
                     }).join();

computation/src/main/java/com/powsybl/computation/ExecutionEnvironment.java

@@ -18,6 +18,7 @@
  *   <li>a map of environment variables</li>
  *   <li>a prefix for the execution working directory</li>
  *   <li>a debug indicator</li>
+     <li>the directory where execution files will be dumped</li>
  * </ul>
  *
  * @author Geoffroy Jamgotchian {@literal <geoffroy.jamgotchian at rte-france.com>}
@@ -32,12 +33,28 @@ public static ExecutionEnvironment createDefault() {
 
     private String workingDirPrefix;
 
+    /**
+     * If debug=true, execution files generated for and by the binary model will not be removed after the computation. There are available in the workingDir.
+     * If debug=false, the workingDir which contains execution files is removed after computation.
+     */
     private boolean debug;
 
-    public ExecutionEnvironment(Map<String, String> variables, String workingDirPrefix, boolean debug) {
+    /**
+     * If dumpDir is provided, in/out execution files are dumped in dumpDir.
+     * If dumpDir=null, nothing is dumped.
+     * debug and dumpDir have fully independent behaviors.
+     */
+    private String dumpDir;
+
+    public ExecutionEnvironment(Map<String, String> variables, String workingDirPrefix, boolean debug, String dumpDir) {
         this.variables = Objects.requireNonNull(variables);
         this.workingDirPrefix = Objects.requireNonNull(workingDirPrefix);
         this.debug = debug;
+        this.dumpDir = dumpDir;
+    }
+
+    public ExecutionEnvironment(Map<String, String> variables, String workingDirPrefix, boolean debug) {
+        this(variables, workingDirPrefix, debug, null);
     }
 
     public Map<String, String> getVariables() {
@@ -66,4 +83,8 @@ public ExecutionEnvironment setDebug(boolean debug) {
         this.debug = debug;
         return this;
     }
+
+    public String getDumpDir() {
+        return dumpDir;
+    }
 }

docs/simulation/dynamic/configuration.md

@@ -26,6 +26,7 @@ You may configure some generic parameters for all implementations:
 dynamic-simulation-default-parameters:
     startTime: 0
     stopTime: 1
+    debugDir: /tmp/debugDir
 ```
 
 The parameters may also be overridden with a JSON file, in which case the configuration will look like:
@@ -34,6 +35,7 @@ The parameters may also be overridden with a JSON file, in which case the config
   "version" : "1.0",
   "startTime" : 0,
   "stopTime" : 1,
+  "debugDir": "/tmp/debugDir",
   "extensions" : {
     ...
   }
@@ -48,6 +50,9 @@ The parameters may also be overridden with a JSON file, in which case the config
 **stopTime**  
 `stopTime` defines when the simulation stops, in seconds. The default value of this property is `1`.
 
+**debugDir**
+This property specifies the directory path where debug files will be dumped. If `null`, no file will be dumped.
+
 ### Specific parameters
 Some implementations use specific parameters that can be defined in the configuration file or in the JSON parameters file:
 - [Dynawo](inv:powsybldynawo:*:*#dynamic_simulation/configuration)
@@ -59,12 +64,14 @@ Some implementations use specific parameters that can be defined in the configur
 dynamic-simulation-default-parameters:
   startTime: 0
   stopTime: 3600
+  debugDir: /tmp/debugDir
 ```
 
 **XML configuration:**
 ```xml
 <dynamic-simulation-default-parameters>
   <startTime>0</startTime>
   <stopTime>3600</stopTime>
+  <debugDir>/tmp/debugDir</debugDir>
 </dynamic-simulation-default-parameters>
 ```

docs/simulation/dynamic_security/configuration.md

@@ -30,6 +30,7 @@ dynamic-simulation-default-parameters:
 
 dynamic-security-analysis-default-parameters:
   contingencies-start-time: 10
+  debugDir: /tmp/debugDir
 ```
 
 The parameters may also be overridden with a JSON file, in which case the configuration will look like:
@@ -43,7 +44,8 @@ The parameters may also be overridden with a JSON file, in which case the config
   },
   "contingencies-parameters" : {
     "contingencies-start-time" : 5.5
-  }
+  },
+  "debugDir": "/tmp/debugDir"
 }
 ```
 
@@ -52,17 +54,22 @@ The parameters may also be overridden with a JSON file, in which case the config
 **contingencies-start-time**  
 `contingencies-start-time` defines when the contingencies start, in seconds. The default value of this property is `5`.
 
+**debugDir**
+This property specifies the directory path where debug files will be dumped. If `null`, no file will be dumped.
+
 ### Examples
 
 **YAML configuration:**
 ```yaml
 dynamic-security-analysis-default-parameters:
   contingencies-start-time: 10
+  debugDir: /tmp/debugDir
 ```
 
 **XML configuration:**
 ```xml
 <dynamic-security-analysis-default-parameters>
   <contingencies-start-time>10</contingencies-start-time>
+  <debugDir>/tmp/debugDir</debugDir>
 </dynamic-security-analysis-default-parameters>
 ```

docs/simulation/shortcircuit/parameters.md

@@ -17,6 +17,8 @@ short-circuit-parameters:
   with-neutral-position: true
   initial-voltage-profile-mode: CONFIGURED
   voltage-ranges: /path/to/voltage/ranges/file
+  detailedReport: true
+  debugDir: null
 ```
 
 Available parameters in the short-circuit API are stored in `com.powsybl.shortcircuit.ShortCircuitParameters`. They are all optional.
@@ -136,6 +138,13 @@ Here is an example of this JSON file:
 ]
 ````
 
+**detailedReport**
+This property indicates whether the computation should produce detailed reporting. If `true`, detailed reports are returned.
+If `false`, summarized reports are returned.
+
+**debugDir**
+This property specifies the directory path where debug files will be dumped. If `null`, no file will be dumped.
+
 (shortcircuit-fault-parameters)=
 ## FaultParameters
 

dynamic-security-analysis/src/main/java/com/powsybl/security/dynamic/DynamicSecurityAnalysisParameters.java

@@ -26,10 +26,13 @@
  */
 public class DynamicSecurityAnalysisParameters extends AbstractExtendable<DynamicSecurityAnalysisParameters> {
 
-    public static final String VERSION = "1.0";
+    // VERSION = 1.0 dynamicSimulationParameters, contingenciesParameters
+    // VERSION = 1.1 debugDir
+    public static final String VERSION = "1.1";
 
     private DynamicSimulationParameters dynamicSimulationParameters = new DynamicSimulationParameters();
     private ContingenciesParameters contingenciesParameters = new ContingenciesParameters();
+    private String debugDir;
 
     public static class ContingenciesParameters {
 
@@ -112,6 +115,15 @@ public DynamicSecurityAnalysisParameters setDynamicContingenciesParameters(Conti
         return this;
     }
 
+    public String getDebugDir() {
+        return debugDir;
+    }
+
+    public DynamicSecurityAnalysisParameters setDebugDir(String debugDir) {
+        this.debugDir = debugDir;
+        return this;
+    }
+
     public void write(Path parametersPath) {
         JsonDynamicSecurityAnalysisParameters.write(this, parametersPath);
     }

dynamic-security-analysis/src/main/java/com/powsybl/security/dynamic/json/DynamicSecurityAnalysisParametersDeserializer.java

@@ -27,6 +27,9 @@
  */
 public class DynamicSecurityAnalysisParametersDeserializer extends StdDeserializer<DynamicSecurityAnalysisParameters> {
 
+    private static final String CONTEXT_NAME = "DynamicSecurityAnalysisParameters";
+    private static final String TAG = "Tag: ";
+
     DynamicSecurityAnalysisParametersDeserializer() {
         super(DynamicSecurityAnalysisParameters.class);
     }
@@ -56,6 +59,11 @@ public DynamicSecurityAnalysisParameters deserialize(JsonParser parser, Deserial
                             parser,
                             DynamicSecurityAnalysisParameters.ContingenciesParameters.class));
                     break;
+                case "debugDir":
+                    JsonUtil.assertGreaterOrEqualThanReferenceVersion(CONTEXT_NAME, TAG + parser.currentName(), version, "1.1");
+                    parser.nextToken();
+                    parameters.setDebugDir(parser.readValueAs(String.class));
+                    break;
                 case "extensions":
                     parser.nextToken();
                     extensions = JsonUtil.updateExtensions(parser, deserializationContext, getExtensionSerializers()::get, parameters);
@@ -64,10 +72,7 @@ public DynamicSecurityAnalysisParameters deserialize(JsonParser parser, Deserial
                     throw new IllegalStateException("Unexpected field: " + parser.currentName());
             }
         }
-        if (version == null || !version.equals("1.0")) {
-            //Only 1.0 version is supported for now
-            throw new IllegalStateException("Version different than 1.0 not supported.");
-        }
+
         extensions.forEach(extension -> parameters.addExtension((Class) extension.getClass(), extension));
         return parameters;
     }

dynamic-security-analysis/src/main/java/com/powsybl/security/dynamic/json/DynamicSecurityAnalysisParametersSerializer.java

@@ -32,6 +32,7 @@ public void serialize(DynamicSecurityAnalysisParameters parameters, JsonGenerato
         jsonGenerator.writeFieldName("dynamic-simulation-parameters");
         JsonDynamicSimulationParameters.serialize(parameters.getDynamicSimulationParameters(), jsonGenerator, serializerProvider);
         serializerProvider.defaultSerializeField("contingencies-parameters", parameters.getDynamicContingenciesParameters(), jsonGenerator);
+        JsonUtil.writeOptionalStringField(jsonGenerator, "debugDir", parameters.getDebugDir());
         JsonUtil.writeExtensions(parameters, jsonGenerator, serializerProvider, JsonDynamicSecurityAnalysisParameters.getExtensionSerializers()::get);
         jsonGenerator.writeEndObject();
     }