From ceccbac0a2ba9dd60f269a246187094a303ea041 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Mon, 9 Feb 2026 16:17:06 +0530 Subject: [PATCH 1/7] Enhance workflow validation with source file and port checks - Add optional --source flag to validate command for file existence checking - Implement cycle detection to identify control loops in workflows - Add ZMQ port conflict detection and validation - Warn about reserved ports (< 1024) and invalid port ranges - Expand test coverage with 6 new comprehensive test cases - Update CLI documentation with new validation features This improves the user experience by catching common configuration errors before workflow execution, reducing runtime failures. --- concore_cli/README.md | 9 ++- concore_cli/cli.py | 5 +- concore_cli/commands/validate.py | 99 ++++++++++++++++++++++++++- tests/test_graph.py | 113 +++++++++++++++++++++++++++++++ 4 files changed, 222 insertions(+), 4 deletions(-) diff --git a/concore_cli/README.md b/concore_cli/README.md index e29c7657..e3b55b68 100644 --- a/concore_cli/README.md +++ b/concore_cli/README.md @@ -72,16 +72,23 @@ concore run workflow.graphml --source ./src --output ./build --auto-build Validates a GraphML workflow file before running. +**Options:** +- `-s, --source ` - Source directory to verify file references exist + Checks: - Valid XML structure - GraphML format compliance - Node and edge definitions - File references and naming conventions -- ZMQ vs file-based communication +- Source file existence (when --source provided) +- ZMQ port conflicts and reserved ports +- Circular dependencies (warns for control loops) +- Edge connectivity **Example:** ```bash concore validate workflow.graphml +concore validate workflow.graphml --source ./src ``` ### `concore status` diff --git a/concore_cli/cli.py b/concore_cli/cli.py index f7144533..4db6068f 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -47,10 +47,11 @@ def run(workflow_file, source, output, type, auto_build): @cli.command() @click.argument('workflow_file', type=click.Path(exists=True)) -def validate(workflow_file): +@click.option('--source', '-s', type=click.Path(exists=True), help='Source directory to check file references') +def validate(workflow_file, source): """Validate a workflow file""" try: - validate_workflow(workflow_file, console) + validate_workflow(workflow_file, console, source) except Exception as e: console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index fa1ea184..7b34d722 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -5,7 +5,7 @@ import re import xml.etree.ElementTree as ET -def validate_workflow(workflow_file, console): +def validate_workflow(workflow_file, console, source_dir=None): workflow_path = Path(workflow_file) console.print(f"[cyan]Validating:[/cyan] {workflow_path.name}") @@ -138,6 +138,12 @@ def validate_workflow(workflow_file, console): if file_edges > 0: info.append(f"File-based edges: {file_edges}") + if source_dir: + _check_source_files(soup, Path(source_dir), errors, warnings) + + _check_cycles(soup, errors, warnings) + _check_zmq_ports(soup, errors, warnings) + show_results(console, errors, warnings, info) except FileNotFoundError: @@ -145,6 +151,97 @@ def validate_workflow(workflow_file, console): except Exception as e: console.print(f"[red]Validation failed:[/red] {str(e)}") +def _check_source_files(soup, source_path, errors, warnings): + nodes = soup.find_all('node') + + for node in nodes: + label_tag = node.find('y:NodeLabel') or node.find('NodeLabel') + if not label_tag or not label_tag.text: + continue + + label = label_tag.text.strip() + if ':' not in label: + continue + + parts = label.split(':') + if len(parts) != 2: + continue + + _, filename = parts + if not filename: + continue + + file_path = source_path / filename + if not file_path.exists(): + errors.append(f"Source file not found: {filename}") + +def _check_cycles(soup, errors, warnings): + nodes = soup.find_all('node') + edges = soup.find_all('edge') + + node_ids = [node.get('id') for node in nodes if node.get('id')] + if not node_ids: + return + + graph = {nid: [] for nid in node_ids} + for edge in edges: + source = edge.get('source') + target = edge.get('target') + if source and target and source in graph: + graph[source].append(target) + + def has_cycle_from(start, visited, rec_stack): + visited.add(start) + rec_stack.add(start) + + for neighbor in graph.get(start, []): + if neighbor not in visited: + if has_cycle_from(neighbor, visited, rec_stack): + return True + elif neighbor in rec_stack: + return True + + rec_stack.remove(start) + return False + + visited = set() + for node_id in node_ids: + if node_id not in visited: + if has_cycle_from(node_id, visited, set()): + warnings.append("Workflow contains cycles (expected for control loops)") + return + +def _check_zmq_ports(soup, errors, warnings): + edges = soup.find_all('edge') + port_pattern = re.compile(r"0x([a-fA-F0-9]+)_(\S+)") + + ports_used = {} + + for edge in edges: + label_tag = edge.find('y:EdgeLabel') or edge.find('EdgeLabel') + if not label_tag or not label_tag.text: + continue + + match = port_pattern.match(label_tag.text.strip()) + if not match: + continue + + port_hex = match.group(1) + port_name = match.group(2) + port_num = int(port_hex, 16) + + if port_num in ports_used: + existing_name = ports_used[port_num] + if existing_name != port_name: + errors.append(f"Port conflict: 0x{port_hex} used for both '{existing_name}' and '{port_name}'") + else: + ports_used[port_num] = port_name + + if port_num < 1024: + warnings.append(f"Port {port_num} (0x{port_hex}) is in reserved range (< 1024)") + elif port_num > 65535: + errors.append(f"Invalid port number: {port_num} (0x{port_hex}) exceeds maximum (65535)") + def show_results(console, errors, warnings, info): if errors: console.print("[red]✗ Validation failed[/red]\n") diff --git a/tests/test_graph.py b/tests/test_graph.py index 97102dce..7489e112 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -128,6 +128,119 @@ def test_validate_valid_graph(self): self.assertIn('Validation passed', result.output) self.assertIn('Workflow is valid', result.output) + + def test_validate_missing_source_file(self): + content = ''' + + + + n0:missing.py + + + + ''' + filepath = self.create_graph_file('workflow.graphml', content) + source_dir = Path(self.temp_dir) / 'src' + source_dir.mkdir() + + result = self.runner.invoke(cli, ['validate', filepath, '--source', str(source_dir)]) + + self.assertIn('Validation failed', result.output) + self.assertIn('Source file not found: missing.py', result.output) + + def test_validate_with_existing_source_file(self): + content = ''' + + + + n0:exists.py + + + + ''' + filepath = self.create_graph_file('workflow.graphml', content) + source_dir = Path(self.temp_dir) / 'src' + source_dir.mkdir() + (source_dir / 'exists.py').write_text('print("hello")') + + result = self.runner.invoke(cli, ['validate', filepath, '--source', str(source_dir)]) + + self.assertIn('Validation passed', result.output) + + def test_validate_zmq_port_conflict(self): + content = ''' + + + + n0:script1.py + + + n1:script2.py + + + 0x1234_portA + + + 0x1234_portB + + + + ''' + filepath = self.create_graph_file('conflict.graphml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('Validation failed', result.output) + self.assertIn('Port conflict', result.output) + + def test_validate_reserved_port(self): + content = ''' + + + + n0:script1.py + + + n1:script2.py + + + 0x50_data + + + + ''' + filepath = self.create_graph_file('reserved.graphml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('Port 80', result.output) + self.assertIn('reserved range', result.output) + + def test_validate_cycle_detection(self): + content = ''' + + + + n0:controller.py + + + n1:plant.py + + + control_signal + + + sensor_data + + + + ''' + filepath = self.create_graph_file('cycle.graphml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('cycles', result.output) + self.assertIn('control loops', result.output) if __name__ == '__main__': unittest.main() \ No newline at end of file From a39e30a4b0fa93394aa993ad6b8e614f3c5d6688 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Mon, 9 Feb 2026 16:28:02 +0530 Subject: [PATCH 2/7] Address Copilot review feedback - Add explicit validation for port 0 (invalid) - Enforce directory-only input for --source option - Use warnings parameter in _check_source_files for clarity - Add test coverage for port 0 and port > 65535 edge cases - Reorder port validation to check range before conflicts --- concore_cli/cli.py | 7 ++++- concore_cli/commands/validate.py | 11 ++++++-- tests/test_graph.py | 46 ++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/concore_cli/cli.py b/concore_cli/cli.py index 4db6068f..f9e35f0c 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -47,7 +47,12 @@ def run(workflow_file, source, output, type, auto_build): @cli.command() @click.argument('workflow_file', type=click.Path(exists=True)) -@click.option('--source', '-s', type=click.Path(exists=True), help='Source directory to check file references') +@click.option( + '--source', + '-s', + type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path), + help='Source directory to check file references', +) def validate(workflow_file, source): """Validate a workflow file""" try: diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index 7b34d722..18456329 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -161,10 +161,12 @@ def _check_source_files(soup, source_path, errors, warnings): label = label_tag.text.strip() if ':' not in label: + warnings.append(f"Skipping node with invalid label format (expected 'ID:filename')") continue parts = label.split(':') if len(parts) != 2: + warnings.append(f"Skipping node '{label}' with invalid format") continue _, filename = parts @@ -230,6 +232,13 @@ def _check_zmq_ports(soup, errors, warnings): port_name = match.group(2) port_num = int(port_hex, 16) + if port_num < 1: + errors.append(f"Invalid port number: {port_num} (0x{port_hex}) must be at least 1") + continue + elif port_num > 65535: + errors.append(f"Invalid port number: {port_num} (0x{port_hex}) exceeds maximum (65535)") + continue + if port_num in ports_used: existing_name = ports_used[port_num] if existing_name != port_name: @@ -239,8 +248,6 @@ def _check_zmq_ports(soup, errors, warnings): if port_num < 1024: warnings.append(f"Port {port_num} (0x{port_hex}) is in reserved range (< 1024)") - elif port_num > 65535: - errors.append(f"Invalid port number: {port_num} (0x{port_hex}) exceeds maximum (65535)") def show_results(console, errors, warnings, info): if errors: diff --git a/tests/test_graph.py b/tests/test_graph.py index 7489e112..483fd06f 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -241,6 +241,52 @@ def test_validate_cycle_detection(self): self.assertIn('cycles', result.output) self.assertIn('control loops', result.output) + + def test_validate_port_zero(self): + content = ''' + + + + n0:script1.py + + + n1:script2.py + + + 0x0_invalid + + + + ''' + filepath = self.create_graph_file('port_zero.graphml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('Validation failed', result.output) + self.assertIn('must be at least 1', result.output) + + def test_validate_port_exceeds_maximum(self): + content = ''' + + + + n0:script1.py + + + n1:script2.py + + + 0x10000_toobig + + + + ''' + filepath = self.create_graph_file('port_max.graphml', content) + + result = self.runner.invoke(cli, ['validate', filepath]) + + self.assertIn('Validation failed', result.output) + self.assertIn('exceeds maximum (65535)', result.output) if __name__ == '__main__': unittest.main() \ No newline at end of file From ad0f393c45ad3498c23fbf1e6b30a68a2ba79b89 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Sun, 15 Feb 2026 04:00:48 +0530 Subject: [PATCH 3/7] Fix --- mkconcore.py | 54 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/mkconcore.py b/mkconcore.py index 400475d6..0f519266 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -77,7 +77,8 @@ def safe_name(value, context): """ Validates that the input string does not contain characters dangerous - for filesystem paths or shell command injection. + for simple names (labels, filenames without paths). + Use safe_path() for validating directory/file paths. """ if not value: raise ValueError(f"{context} cannot be empty") @@ -86,22 +87,36 @@ def safe_name(value, context): raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") return value -MKCONCORE_VER = "22-09-18" - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) - -def _resolve_concore_path(): - script_concore = os.path.join(SCRIPT_DIR, "concore.py") - if os.path.exists(script_concore): - return SCRIPT_DIR - cwd_concore = os.path.join(os.getcwd(), "concore.py") - if os.path.exists(cwd_concore): - return os.getcwd() - return SCRIPT_DIR - -GRAPHML_FILE = sys.argv[1] -TRIMMED_LOGS = True -CONCOREPATH = _resolve_concore_path() +def safe_path(value, context): + """ + Validates that a path string does not contain characters dangerous for shell command injection. + Unlike safe_name(), this allows path separators (/ and \) but still blocks dangerous shell metacharacters. + """ + if not value: + raise ValueError(f"{context} cannot be empty") + # Allow path separators but block control characters and shell metacharacters + # Blocks: control chars, *, ?, <, >, |, ;, &, $, `, ', ", (, ) + # Allows: /, \, -, _, ., alphanumeric, spaces, : + if re.search(r'[\x00-\x1F\x7F*?"<>|;&`$\'()]', value): + raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") + return value + +MKCONCORE_VER = "22-09-18" + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +def _resolve_concore_path(): + script_concore = os.path.join(SCRIPT_DIR, "concore.py") + if os.path.exists(script_concore): + return SCRIPT_DIR + cwd_concore = os.path.join(os.getcwd(), "concore.py") + if os.path.exists(cwd_concore): + return os.getcwd() + return SCRIPT_DIR + +GRAPHML_FILE = sys.argv[1] +TRIMMED_LOGS = True +CONCOREPATH = _resolve_concore_path() CPPWIN = "g++" #Windows C++ 6/22/21 CPPEXE = "g++" #Ubuntu/macOS C++ 6/22/21 VWIN = "iverilog" #Windows verilog 6/25/21 @@ -142,8 +157,9 @@ def _resolve_concore_path(): sourcedir = sys.argv[2] outdir = sys.argv[3] -# Validate outdir argument -safe_name(outdir, "Output directory argument") +# Validate path arguments +safe_path(outdir, "Output directory argument") +safe_path(sourcedir, "Source directory argument") if not os.path.isdir(sourcedir): logging.error(f"{sourcedir} does not exist") From 59c76b6b942f3f3f1eb9257a8111c9d8f0257084 Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Sun, 15 Feb 2026 04:58:02 +0530 Subject: [PATCH 4/7] Merge upstream/dev --- abort_rebase.py | 12 +++++++ complete_push.py | 61 ++++++++++++++++++++++++++++++++ concore_cli/cli.py | 11 +++--- concore_cli/commands/validate.py | 60 +++++++++++++------------------ fix_and_push.bat | 8 +++++ mkconcore.py | 25 ++++++++++++- tests/test_graph.py | 2 +- 7 files changed, 135 insertions(+), 44 deletions(-) create mode 100644 abort_rebase.py create mode 100644 complete_push.py create mode 100644 fix_and_push.bat diff --git a/abort_rebase.py b/abort_rebase.py new file mode 100644 index 00000000..88dcb2fb --- /dev/null +++ b/abort_rebase.py @@ -0,0 +1,12 @@ +import subprocess +import os + +os.chdir(r'C:\Users\Sahil\concore') + +# Abort rebase +subprocess.run(['git', 'rebase', '--abort'], capture_output=True) + +# Check status +result = subprocess.run(['git', 'status'], capture_output=True, text=True) +print(result.stdout) +print(result.stderr) diff --git a/complete_push.py b/complete_push.py new file mode 100644 index 00000000..16c03613 --- /dev/null +++ b/complete_push.py @@ -0,0 +1,61 @@ +import subprocess +import sys +import os + +os.chdir(r'C:\Users\Sahil\concore') + +print("Aborting rebase and recovering branch state...") + +# Method 1: Direct HEAD manipulation +with open('.git/HEAD', 'w') as f: + f.write('ref: refs/heads/feature/enhanced-workflow-validation\n') + +# Remove rebase state +import shutil +try: + shutil.rmtree('.git/rebase-merge') + print("✓ Removed rebase-merge directory") +except: + pass + +try: + os.remove('.git/REBASE_HEAD') + print("✓ Removed REBASE_HEAD") +except: + pass + +# Reset to original HEAD +result = subprocess.run(['git', 'reset', '--hard', 'ad0f393'], capture_output=True, text=True) +print(result.stdout) +if result.stderr: + print(result.stderr) + +# Check status +result = subprocess.run(['git', 'status', '--short'], capture_output=True, text=True) +print("\nCurrent status:") +print(result.stdout) + +# Fetch upstream +print("\nFetching upstream...") +result = subprocess.run(['git', 'fetch', 'upstream'], capture_output=True, text=True) +if result.returncode != 0: + print(result.stderr) + +# Merge upstream/dev +print("\nMerging upstream/dev...") +result = subprocess.run(['git', 'merge', 'upstream/dev', '-m', 'Merge upstream/dev'], capture_output=True, text=True) +print(result.stdout) +if result.returncode != 0: + print("Merge conflicts or error:") + print(result.stderr) + sys.exit(1) + +# Push +print("\nPushing to origin...") +result = subprocess.run(['git', 'push', 'origin', 'feature/enhanced-workflow-validation', '--force-with-lease'], capture_output=True, text=True) +print(result.stdout) +if result.returncode != 0: + print(result.stderr) + sys.exit(1) + +print("\n✓ Successfully pushed!") diff --git a/concore_cli/cli.py b/concore_cli/cli.py index f9e35f0c..e6fc061d 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -47,16 +47,13 @@ def run(workflow_file, source, output, type, auto_build): @cli.command() @click.argument('workflow_file', type=click.Path(exists=True)) -@click.option( - '--source', - '-s', - type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path), - help='Source directory to check file references', -) +@click.option('--source', '-s', default='src', help='Source directory') def validate(workflow_file, source): """Validate a workflow file""" try: - validate_workflow(workflow_file, console, source) + ok = validate_workflow(workflow_file, source, console) + if not ok: + sys.exit(1) except Exception as e: console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) diff --git a/concore_cli/commands/validate.py b/concore_cli/commands/validate.py index 18456329..7cacab99 100644 --- a/concore_cli/commands/validate.py +++ b/concore_cli/commands/validate.py @@ -5,8 +5,9 @@ import re import xml.etree.ElementTree as ET -def validate_workflow(workflow_file, console, source_dir=None): +def validate_workflow(workflow_file, source_dir, console): workflow_path = Path(workflow_file) + source_root = (workflow_path.parent / source_dir) console.print(f"[cyan]Validating:[/cyan] {workflow_path.name}") console.print() @@ -15,31 +16,35 @@ def validate_workflow(workflow_file, console, source_dir=None): warnings = [] info = [] + def finalize(): + show_results(console, errors, warnings, info) + return len(errors) == 0 + try: with open(workflow_path, 'r') as f: content = f.read() if not content.strip(): errors.append("File is empty") - return show_results(console, errors, warnings, info) + return finalize() # strict XML syntax check try: ET.fromstring(content) except ET.ParseError as e: errors.append(f"Invalid XML: {str(e)}") - return show_results(console, errors, warnings, info) + return finalize() try: soup = BeautifulSoup(content, 'xml') except Exception as e: errors.append(f"Invalid XML: {str(e)}") - return show_results(console, errors, warnings, info) + return finalize() root = soup.find('graphml') if not root: errors.append("Not a valid GraphML file - missing root element") - return show_results(console, errors, warnings, info) + return finalize() # check the graph attributes graph = soup.find('graph') @@ -65,6 +70,9 @@ def validate_workflow(workflow_file, console, source_dir=None): else: info.append(f"Found {len(edges)} edge(s)") + if not source_root.exists(): + warnings.append(f"Source directory not found: {source_root}") + node_labels = [] for node in nodes: #check the node id @@ -84,6 +92,11 @@ def validate_workflow(workflow_file, console, source_dir=None): label = label_tag.text.strip() node_labels.append(label) + # reject shell metacharacters to prevent command injection (#251) + if re.search(r'[;&|`$\'"()\\]', label): + errors.append(f"Node '{label}' contains unsafe shell characters") + continue + if ':' not in label: warnings.append(f"Node '{label}' missing format 'ID:filename'") else: @@ -96,6 +109,10 @@ def validate_workflow(workflow_file, console, source_dir=None): errors.append(f"Node '{label}' has no filename") elif not any(filename.endswith(ext) for ext in ['.py', '.cpp', '.m', '.v', '.java']): warnings.append(f"Node '{label}' has unusual file extension") + elif source_root.exists(): + file_path = source_root / filename + if not file_path.exists(): + errors.append(f"Missing source file: {filename}") else: warnings.append(f"Node {node_id} has no label") except Exception as e: @@ -138,44 +155,17 @@ def validate_workflow(workflow_file, console, source_dir=None): if file_edges > 0: info.append(f"File-based edges: {file_edges}") - if source_dir: - _check_source_files(soup, Path(source_dir), errors, warnings) - _check_cycles(soup, errors, warnings) _check_zmq_ports(soup, errors, warnings) - show_results(console, errors, warnings, info) + return finalize() except FileNotFoundError: console.print(f"[red]Error:[/red] File not found: {workflow_path}") + return False except Exception as e: console.print(f"[red]Validation failed:[/red] {str(e)}") - -def _check_source_files(soup, source_path, errors, warnings): - nodes = soup.find_all('node') - - for node in nodes: - label_tag = node.find('y:NodeLabel') or node.find('NodeLabel') - if not label_tag or not label_tag.text: - continue - - label = label_tag.text.strip() - if ':' not in label: - warnings.append(f"Skipping node with invalid label format (expected 'ID:filename')") - continue - - parts = label.split(':') - if len(parts) != 2: - warnings.append(f"Skipping node '{label}' with invalid format") - continue - - _, filename = parts - if not filename: - continue - - file_path = source_path / filename - if not file_path.exists(): - errors.append(f"Source file not found: {filename}") + return False def _check_cycles(soup, errors, warnings): nodes = soup.find_all('node') diff --git a/fix_and_push.bat b/fix_and_push.bat new file mode 100644 index 00000000..f1a74e3c --- /dev/null +++ b/fix_and_push.bat @@ -0,0 +1,8 @@ +@echo off +cd C:\Users\Sahil\concore +rmdir /s /q .git\rebase-merge 2>nul +del /f .git\REBASE_HEAD 2>nul +git reset --hard ad0f393 +git fetch upstream +git merge upstream/dev -m "Merge upstream/dev" +git push origin feature/enhanced-workflow-validation --force-with-lease diff --git a/mkconcore.py b/mkconcore.py index 0f519266..45a4cc62 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -101,6 +101,25 @@ def safe_path(value, context): raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") return value +def safe_relpath(value, context): + """ + Allow relative subpaths while blocking traversal and absolute/drive paths. + Used for node source file paths that may contain subdirectories. + """ + if not value: + raise ValueError(f"{context} cannot be empty") + normalized = value.replace("\\", "/") + safe_path(normalized, context) + if normalized.startswith("/") or normalized.startswith("~"): + raise ValueError(f"Unsafe {context}: absolute paths are not allowed.") + if re.match(r"^[A-Za-z]:", normalized): + raise ValueError(f"Unsafe {context}: drive paths are not allowed.") + if ":" in normalized: + raise ValueError(f"Unsafe {context}: ':' is not allowed in relative paths.") + if any(part in ("", "..") for part in normalized.split("/")): + raise ValueError(f"Unsafe {context}: invalid path segment.") + return normalized + MKCONCORE_VER = "22-09-18" SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -251,7 +270,8 @@ def _resolve_concore_path(): if ':' in node_label: container_part, source_part = node_label.split(':', 1) safe_name(container_part, f"Node container name '{container_part}'") - safe_name(source_part, f"Node source file '{source_part}'") + source_part = safe_relpath(source_part, f"Node source file '{source_part}'") + node_label = f"{container_part}:{source_part}" else: safe_name(node_label, f"Node label '{node_label}'") # Explicitly reject incorrect format to prevent later crashes and ambiguity @@ -447,6 +467,9 @@ def _resolve_concore_path(): dockername, langext = sourcecode, "" script_target_path = os.path.join(outdir, "src", sourcecode) + script_target_parent = os.path.dirname(script_target_path) + if script_target_parent: + os.makedirs(script_target_parent, exist_ok=True) # If the script was specialized, it's already in outdir/src. If not, copy from sourcedir. if node_id_key not in node_edge_params: diff --git a/tests/test_graph.py b/tests/test_graph.py index 11fcbd99..6aef0d3c 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -163,7 +163,7 @@ def test_validate_missing_source_file(self): result = self.runner.invoke(cli, ['validate', filepath, '--source', str(source_dir)]) self.assertIn('Validation failed', result.output) - self.assertIn('Source file not found: missing.py', result.output) + self.assertIn('Missing source file', result.output) def test_validate_with_existing_source_file(self): content = ''' From 8cc3d50d3403d0aafe8e5fbe7035719a46ad8cdd Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Sun, 15 Feb 2026 04:58:30 +0530 Subject: [PATCH 5/7] Remove temporary helper scripts --- abort_rebase.py | 12 ---------- complete_push.py | 61 ------------------------------------------------ fix_and_push.bat | 8 ------- 3 files changed, 81 deletions(-) delete mode 100644 abort_rebase.py delete mode 100644 complete_push.py delete mode 100644 fix_and_push.bat diff --git a/abort_rebase.py b/abort_rebase.py deleted file mode 100644 index 88dcb2fb..00000000 --- a/abort_rebase.py +++ /dev/null @@ -1,12 +0,0 @@ -import subprocess -import os - -os.chdir(r'C:\Users\Sahil\concore') - -# Abort rebase -subprocess.run(['git', 'rebase', '--abort'], capture_output=True) - -# Check status -result = subprocess.run(['git', 'status'], capture_output=True, text=True) -print(result.stdout) -print(result.stderr) diff --git a/complete_push.py b/complete_push.py deleted file mode 100644 index 16c03613..00000000 --- a/complete_push.py +++ /dev/null @@ -1,61 +0,0 @@ -import subprocess -import sys -import os - -os.chdir(r'C:\Users\Sahil\concore') - -print("Aborting rebase and recovering branch state...") - -# Method 1: Direct HEAD manipulation -with open('.git/HEAD', 'w') as f: - f.write('ref: refs/heads/feature/enhanced-workflow-validation\n') - -# Remove rebase state -import shutil -try: - shutil.rmtree('.git/rebase-merge') - print("✓ Removed rebase-merge directory") -except: - pass - -try: - os.remove('.git/REBASE_HEAD') - print("✓ Removed REBASE_HEAD") -except: - pass - -# Reset to original HEAD -result = subprocess.run(['git', 'reset', '--hard', 'ad0f393'], capture_output=True, text=True) -print(result.stdout) -if result.stderr: - print(result.stderr) - -# Check status -result = subprocess.run(['git', 'status', '--short'], capture_output=True, text=True) -print("\nCurrent status:") -print(result.stdout) - -# Fetch upstream -print("\nFetching upstream...") -result = subprocess.run(['git', 'fetch', 'upstream'], capture_output=True, text=True) -if result.returncode != 0: - print(result.stderr) - -# Merge upstream/dev -print("\nMerging upstream/dev...") -result = subprocess.run(['git', 'merge', 'upstream/dev', '-m', 'Merge upstream/dev'], capture_output=True, text=True) -print(result.stdout) -if result.returncode != 0: - print("Merge conflicts or error:") - print(result.stderr) - sys.exit(1) - -# Push -print("\nPushing to origin...") -result = subprocess.run(['git', 'push', 'origin', 'feature/enhanced-workflow-validation', '--force-with-lease'], capture_output=True, text=True) -print(result.stdout) -if result.returncode != 0: - print(result.stderr) - sys.exit(1) - -print("\n✓ Successfully pushed!") diff --git a/fix_and_push.bat b/fix_and_push.bat deleted file mode 100644 index f1a74e3c..00000000 --- a/fix_and_push.bat +++ /dev/null @@ -1,8 +0,0 @@ -@echo off -cd C:\Users\Sahil\concore -rmdir /s /q .git\rebase-merge 2>nul -del /f .git\REBASE_HEAD 2>nul -git reset --hard ad0f393 -git fetch upstream -git merge upstream/dev -m "Merge upstream/dev" -git push origin feature/enhanced-workflow-validation --force-with-lease From ab15e7abe264c79ea8a671c6c5338cd9f4037f4c Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Sun, 15 Feb 2026 05:05:39 +0530 Subject: [PATCH 6/7] Fix syntax warning: escape backslash in docstring --- mkconcore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkconcore.py b/mkconcore.py index 45a4cc62..5e830f7a 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -90,7 +90,7 @@ def safe_name(value, context): def safe_path(value, context): """ Validates that a path string does not contain characters dangerous for shell command injection. - Unlike safe_name(), this allows path separators (/ and \) but still blocks dangerous shell metacharacters. + Unlike safe_name(), this allows path separators (/ and \\) but still blocks dangerous shell metacharacters. """ if not value: raise ValueError(f"{context} cannot be empty") From 5ffd847e590fb748c7b91b661491475133663adf Mon Sep 17 00:00:00 2001 From: Sahil Lenka Date: Sun, 15 Feb 2026 05:28:19 +0530 Subject: [PATCH 7/7] Properly merge upstream/dev --- concore_cli/cli.py | 22 ++- mkconcore.py | 398 +++++++++++++++++++++++++-------------------- 2 files changed, 235 insertions(+), 185 deletions(-) diff --git a/concore_cli/cli.py b/concore_cli/cli.py index e6fc061d..615cb7b9 100644 --- a/concore_cli/cli.py +++ b/concore_cli/cli.py @@ -1,19 +1,17 @@ import click from rich.console import Console -from rich.table import Table -from rich.panel import Panel -from rich import print as rprint -import sys import os -from pathlib import Path +import sys from .commands.init import init_project from .commands.run import run_workflow from .commands.validate import validate_workflow from .commands.status import show_status from .commands.stop import stop_all +from .commands.inspect import inspect_workflow console = Console() +DEFAULT_EXEC_TYPE = 'windows' if os.name == 'nt' else 'posix' @click.group() @click.version_option(version='1.0.0', prog_name='concore') @@ -35,7 +33,7 @@ def init(name, template): @click.argument('workflow_file', type=click.Path(exists=True)) @click.option('--source', '-s', default='src', help='Source directory') @click.option('--output', '-o', default='out', help='Output directory') -@click.option('--type', '-t', default='windows', type=click.Choice(['windows', 'posix', 'docker']), help='Execution type') +@click.option('--type', '-t', default=DEFAULT_EXEC_TYPE, type=click.Choice(['windows', 'posix', 'docker']), help='Execution type') @click.option('--auto-build', is_flag=True, help='Automatically run build after generation') def run(workflow_file, source, output, type, auto_build): """Run a concore workflow""" @@ -58,6 +56,18 @@ def validate(workflow_file, source): console.print(f"[red]Error:[/red] {str(e)}") sys.exit(1) +@cli.command() +@click.argument('workflow_file', type=click.Path(exists=True)) +@click.option('--source', '-s', default='src', help='Source directory') +@click.option('--json', 'output_json', is_flag=True, help='Output in JSON format') +def inspect(workflow_file, source, output_json): + """Inspect a workflow file and show its structure""" + try: + inspect_workflow(workflow_file, source, output_json, console) + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) + @cli.command() def status(): """Show running concore processes""" diff --git a/mkconcore.py b/mkconcore.py index 5e830f7a..9457d74e 100644 --- a/mkconcore.py +++ b/mkconcore.py @@ -63,6 +63,7 @@ # - Sets the executable permission (`stat.S_IRWXU`) for the generated scripts on POSIX systems. from bs4 import BeautifulSoup +import atexit import logging import re import sys @@ -74,78 +75,81 @@ import shlex # Added for POSIX shell escaping # input validation helper -def safe_name(value, context): - """ - Validates that the input string does not contain characters dangerous - for simple names (labels, filenames without paths). - Use safe_path() for validating directory/file paths. - """ - if not value: - raise ValueError(f"{context} cannot be empty") - # blocks path traversal (/, \), control characters, and shell metacharacters (*, ?, <, >, |, ;, &, $, `, ', ", (, )) - if re.search(r'[\x00-\x1F\x7F\\/:*?"<>|;&`$\'()]', value): - raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") - return value - -def safe_path(value, context): - """ - Validates that a path string does not contain characters dangerous for shell command injection. - Unlike safe_name(), this allows path separators (/ and \\) but still blocks dangerous shell metacharacters. - """ - if not value: - raise ValueError(f"{context} cannot be empty") - # Allow path separators but block control characters and shell metacharacters - # Blocks: control chars, *, ?, <, >, |, ;, &, $, `, ', ", (, ) - # Allows: /, \, -, _, ., alphanumeric, spaces, : - if re.search(r'[\x00-\x1F\x7F*?"<>|;&`$\'()]', value): - raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") - return value - -def safe_relpath(value, context): - """ - Allow relative subpaths while blocking traversal and absolute/drive paths. - Used for node source file paths that may contain subdirectories. - """ - if not value: - raise ValueError(f"{context} cannot be empty") - normalized = value.replace("\\", "/") - safe_path(normalized, context) - if normalized.startswith("/") or normalized.startswith("~"): - raise ValueError(f"Unsafe {context}: absolute paths are not allowed.") - if re.match(r"^[A-Za-z]:", normalized): - raise ValueError(f"Unsafe {context}: drive paths are not allowed.") - if ":" in normalized: - raise ValueError(f"Unsafe {context}: ':' is not allowed in relative paths.") - if any(part in ("", "..") for part in normalized.split("/")): - raise ValueError(f"Unsafe {context}: invalid path segment.") - return normalized - -MKCONCORE_VER = "22-09-18" - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) - -def _resolve_concore_path(): - script_concore = os.path.join(SCRIPT_DIR, "concore.py") - if os.path.exists(script_concore): - return SCRIPT_DIR - cwd_concore = os.path.join(os.getcwd(), "concore.py") - if os.path.exists(cwd_concore): - return os.getcwd() - return SCRIPT_DIR - -GRAPHML_FILE = sys.argv[1] -TRIMMED_LOGS = True -CONCOREPATH = _resolve_concore_path() -CPPWIN = "g++" #Windows C++ 6/22/21 -CPPEXE = "g++" #Ubuntu/macOS C++ 6/22/21 -VWIN = "iverilog" #Windows verilog 6/25/21 -VEXE = "iverilog" #Ubuntu/macOS verilog 6/25/21 -PYTHONEXE = "python3" #Ubuntu/macOS python3 -PYTHONWIN = "python" #Windows python3 -MATLABEXE = "matlab" #Ubuntu/macOS matlab -MATLABWIN = "matlab" #Windows matlab -OCTAVEEXE = "octave" #Ubuntu/macOS octave -OCTAVEWIN = "octave" #Windows octave +def safe_name(value, context, allow_path=False): + """ + Validates that the input string does not contain characters dangerous + for filesystem paths or shell command injection. + """ + if not value: + raise ValueError(f"{context} cannot be empty") + # blocks control characters and shell metacharacters + # allow path separators and drive colons for full paths when needed + if allow_path: + pattern = r'[\x00-\x1F\x7F*?"<>|;&`$\'()]' + else: + # blocks path traversal (/, \, :) in addition to shell metacharacters + pattern = r'[\x00-\x1F\x7F\\/:*?"<>|;&`$\'()]' + if re.search(pattern, value): + raise ValueError(f"Unsafe {context}: '{value}' contains illegal characters.") + return value + +def safe_relpath(value, context): + """ + Allow relative subpaths while blocking traversal and absolute/drive paths. + """ + if not value: + raise ValueError(f"{context} cannot be empty") + normalized = value.replace("\\", "/") + safe_name(normalized, context, allow_path=True) + if normalized.startswith("/") or normalized.startswith("~"): + raise ValueError(f"Unsafe {context}: absolute paths are not allowed.") + if re.match(r"^[A-Za-z]:", normalized): + raise ValueError(f"Unsafe {context}: drive paths are not allowed.") + if ":" in normalized: + raise ValueError(f"Unsafe {context}: ':' is not allowed in relative paths.") + if any(part in ("", "..") for part in normalized.split("/")): + raise ValueError(f"Unsafe {context}: invalid path segment.") + return normalized + +MKCONCORE_VER = "22-09-18" + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +def _load_tool_config(filepath): + tools = {} + with open(filepath, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + k, v = k.strip(), v.strip() + if v: + tools[k] = v + return tools + +def _resolve_concore_path(): + script_concore = os.path.join(SCRIPT_DIR, "concore.py") + if os.path.exists(script_concore): + return SCRIPT_DIR + cwd_concore = os.path.join(os.getcwd(), "concore.py") + if os.path.exists(cwd_concore): + return os.getcwd() + return SCRIPT_DIR + +GRAPHML_FILE = sys.argv[1] +TRIMMED_LOGS = True +CONCOREPATH = _resolve_concore_path() +CPPWIN = os.environ.get("CONCORE_CPPWIN", "g++") #Windows C++ 6/22/21 +CPPEXE = os.environ.get("CONCORE_CPPEXE", "g++") #Ubuntu/macOS C++ 6/22/21 +VWIN = os.environ.get("CONCORE_VWIN", "iverilog") #Windows verilog 6/25/21 +VEXE = os.environ.get("CONCORE_VEXE", "iverilog") #Ubuntu/macOS verilog 6/25/21 +PYTHONEXE = os.environ.get("CONCORE_PYTHONEXE", "python3") #Ubuntu/macOS python3 +PYTHONWIN = os.environ.get("CONCORE_PYTHONWIN", "python") #Windows python3 +MATLABEXE = os.environ.get("CONCORE_MATLABEXE", "matlab") #Ubuntu/macOS matlab +MATLABWIN = os.environ.get("CONCORE_MATLABWIN", "matlab") #Windows matlab +OCTAVEEXE = os.environ.get("CONCORE_OCTAVEEXE", "octave") #Ubuntu/macOS octave +OCTAVEWIN = os.environ.get("CONCORE_OCTAVEWIN", "octave") #Windows octave M_IS_OCTAVE = False #treat .m as octave MCRPATH = "~/MATLAB/R2021a" #path to local Ubunta Matlab Compiler Runtime DOCKEREXE = "sudo docker"#assume simple docker install @@ -163,22 +167,36 @@ def _resolve_concore_path(): M_IS_OCTAVE = True #treat .m as octave 9/27/21 if os.path.exists(CONCOREPATH+"/concore.mcr"): # 11/12/21 - MCRPATH = open(CONCOREPATH+"/concore.mcr", "r").readline().strip() #path to local Ubunta Matlab Compiler Runtime + with open(CONCOREPATH+"/concore.mcr", "r") as f: + MCRPATH = f.readline().strip() #path to local Ubunta Matlab Compiler Runtime if os.path.exists(CONCOREPATH+"/concore.sudo"): # 12/04/21 - DOCKEREXE = open(CONCOREPATH+"/concore.sudo", "r").readline().strip() #to omit sudo in docker + with open(CONCOREPATH+"/concore.sudo", "r") as f: + DOCKEREXE = f.readline().strip() #to omit sudo in docker if os.path.exists(CONCOREPATH+"/concore.repo"): # 12/04/21 - DOCKEREPO = open(CONCOREPATH+"/concore.repo", "r").readline().strip() #docker id for repo - + with open(CONCOREPATH+"/concore.repo", "r") as f: + DOCKEREPO = f.readline().strip() #docker id for repo + +if os.path.exists(CONCOREPATH+"/concore.tools"): + _tools = _load_tool_config(CONCOREPATH+"/concore.tools") + CPPWIN = _tools.get("CPPWIN", CPPWIN) + CPPEXE = _tools.get("CPPEXE", CPPEXE) + VWIN = _tools.get("VWIN", VWIN) + VEXE = _tools.get("VEXE", VEXE) + PYTHONEXE = _tools.get("PYTHONEXE", PYTHONEXE) + PYTHONWIN = _tools.get("PYTHONWIN", PYTHONWIN) + MATLABEXE = _tools.get("MATLABEXE", MATLABEXE) + MATLABWIN = _tools.get("MATLABWIN", MATLABWIN) + OCTAVEEXE = _tools.get("OCTAVEEXE", OCTAVEEXE) + OCTAVEWIN = _tools.get("OCTAVEWIN", OCTAVEWIN) prefixedgenode = "" sourcedir = sys.argv[2] outdir = sys.argv[3] -# Validate path arguments -safe_path(outdir, "Output directory argument") -safe_path(sourcedir, "Source directory argument") +# Validate outdir argument (allow full paths) +safe_name(outdir, "Output directory argument", allow_path=True) if not os.path.isdir(sourcedir): logging.error(f"{sourcedir} does not exist") @@ -230,6 +248,12 @@ def _resolve_concore_path(): funlock = open("unlock", "w") # 12/4/21 fparams = open("params", "w") # 9/18/22 +def cleanup_script_files(): + for fh in [fbuild, frun, fdebug, fstop, fclear, fmaxtime, funlock, fparams]: + if not fh.closed: + fh.close() +atexit.register(cleanup_script_files) + os.mkdir("src") os.chdir("..") @@ -243,8 +267,8 @@ def _resolve_concore_path(): logging.info(f"MCR path: {MCRPATH}") logging.info(f"Docker repository: {DOCKEREPO}") -f = open(GRAPHML_FILE, "r") -text_str = f.read() +with open(GRAPHML_FILE, "r") as f: + text_str = f.read() soup = BeautifulSoup(text_str, 'xml') @@ -267,15 +291,15 @@ def _resolve_concore_path(): node_label = re.sub(r'(\s+|\n)', ' ', node_label) #Validate node labels - if ':' in node_label: - container_part, source_part = node_label.split(':', 1) - safe_name(container_part, f"Node container name '{container_part}'") - source_part = safe_relpath(source_part, f"Node source file '{source_part}'") - node_label = f"{container_part}:{source_part}" - else: - safe_name(node_label, f"Node label '{node_label}'") - # Explicitly reject incorrect format to prevent later crashes and ambiguity - raise ValueError(f"Invalid node label '{node_label}': expected format 'container:source' with a ':' separator.") + if ':' in node_label: + container_part, source_part = node_label.split(':', 1) + safe_name(container_part, f"Node container name '{container_part}'") + source_part = safe_relpath(source_part, f"Node source file '{source_part}'") + node_label = f"{container_part}:{source_part}" + else: + safe_name(node_label, f"Node label '{node_label}'") + # Explicitly reject incorrect format to prevent later crashes and ambiguity + raise ValueError(f"Invalid node label '{node_label}': expected format 'container:source' with a ':' separator.") nodes_dict[node['id']] = node_label node_id_to_label_map[node['id']] = node_label.split(':')[0] @@ -461,15 +485,15 @@ def _resolve_concore_path(): if not sourcecode: continue - if "." in sourcecode: - dockername, langext = os.path.splitext(sourcecode) - else: - dockername, langext = sourcecode, "" - - script_target_path = os.path.join(outdir, "src", sourcecode) - script_target_parent = os.path.dirname(script_target_path) - if script_target_parent: - os.makedirs(script_target_parent, exist_ok=True) + if "." in sourcecode: + dockername, langext = os.path.splitext(sourcecode) + else: + dockername, langext = sourcecode, "" + + script_target_path = os.path.join(outdir, "src", sourcecode) + script_target_parent = os.path.dirname(script_target_path) + if script_target_parent: + os.makedirs(script_target_parent, exist_ok=True) # If the script was specialized, it's already in outdir/src. If not, copy from sourcedir. if node_id_key not in node_edge_params: @@ -494,121 +518,125 @@ def _resolve_concore_path(): #copy proper concore.py into /src try: if concoretype=="docker": - fsource = open(CONCOREPATH+"/concoredocker.py") + with open(CONCOREPATH+"/concoredocker.py") as fsource: + source_content = fsource.read() else: - fsource = open(CONCOREPATH+"/concore.py") + with open(CONCOREPATH+"/concore.py") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore.py","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) #copy proper concore.hpp into /src 6/22/21 try: if concoretype=="docker": - fsource = open(CONCOREPATH+"/concoredocker.hpp") + with open(CONCOREPATH+"/concoredocker.hpp") as fsource: + source_content = fsource.read() else: - fsource = open(CONCOREPATH+"/concore.hpp") + with open(CONCOREPATH+"/concore.hpp") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore.hpp","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) #copy proper concore.v into /src 6/25/21 try: if concoretype=="docker": - fsource = open(CONCOREPATH+"/concoredocker.v") + with open(CONCOREPATH+"/concoredocker.v") as fsource: + source_content = fsource.read() else: - fsource = open(CONCOREPATH+"/concore.v") + with open(CONCOREPATH+"/concore.v") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore.v","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) #copy mkcompile into /src 5/27/21 try: - fsource = open(CONCOREPATH+"/mkcompile") + with open(CONCOREPATH+"/mkcompile") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/mkcompile","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) os.chmod(outdir+"/src/mkcompile",stat.S_IRWXU) #copy concore*.m into /src 4/2/21 try: #maxtime in matlab 11/22/21 - fsource = open(CONCOREPATH+"/concore_default_maxtime.m") + with open(CONCOREPATH+"/concore_default_maxtime.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_default_maxtime.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: - fsource = open(CONCOREPATH+"/concore_unchanged.m") + with open(CONCOREPATH+"/concore_unchanged.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_unchanged.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: - fsource = open(CONCOREPATH+"/concore_read.m") + with open(CONCOREPATH+"/concore_read.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_read.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: - fsource = open(CONCOREPATH+"/concore_write.m") + with open(CONCOREPATH+"/concore_write.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_write.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: #4/9/21 - fsource = open(CONCOREPATH+"/concore_initval.m") + with open(CONCOREPATH+"/concore_initval.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_initval.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: #11/19/21 - fsource = open(CONCOREPATH+"/concore_iport.m") + with open(CONCOREPATH+"/concore_iport.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_iport.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: #11/19/21 - fsource = open(CONCOREPATH+"/concore_oport.m") + with open(CONCOREPATH+"/concore_oport.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/concore_oport.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) try: # 4/4/21 if concoretype=="docker": - fsource = open(CONCOREPATH+"/import_concoredocker.m") + with open(CONCOREPATH+"/import_concoredocker.m") as fsource: + source_content = fsource.read() else: - fsource = open(CONCOREPATH+"/import_concore.m") + with open(CONCOREPATH+"/import_concore.m") as fsource: + source_content = fsource.read() except (FileNotFoundError, IOError) as e: logging.error(f"{CONCOREPATH} is not correct path to concore: {e}") quit() with open(outdir+"/src/import_concore.m","w") as fcopy: - fcopy.write(fsource.read()) -fsource.close() + fcopy.write(source_content) # --- Generate iport and oport mappings --- logging.info("Generating iport/oport mappings...") @@ -655,47 +683,58 @@ def _resolve_concore_path(): # 4. Write final iport/oport files logging.info("Writing .iport and .oport files...") -for node_label, ports in node_port_mappings.items(): +for node_label, ports in node_port_mappings.items(): try: containername, sourcecode = node_label.split(':', 1) - if not sourcecode or "." not in sourcecode: continue - dockername = os.path.splitext(sourcecode)[0] - with open(os.path.join(outdir, "src", f"{dockername}.iport"), "w") as fport: - fport.write(str(ports['iport']).replace("'" + prefixedgenode, "'")) - with open(os.path.join(outdir, "src", f"{dockername}.oport"), "w") as fport: - fport.write(str(ports['oport']).replace("'" + prefixedgenode, "'")) + if not sourcecode or "." not in sourcecode: continue + dockername = os.path.splitext(sourcecode)[0] + iport_path = os.path.join(outdir, "src", f"{dockername}.iport") + oport_path = os.path.join(outdir, "src", f"{dockername}.oport") + iport_parent = os.path.dirname(iport_path) + if iport_parent: + os.makedirs(iport_parent, exist_ok=True) + with open(iport_path, "w") as fport: + fport.write(str(ports['iport']).replace("'" + prefixedgenode, "'")) + with open(oport_path, "w") as fport: + fport.write(str(ports['oport']).replace("'" + prefixedgenode, "'")) except ValueError: continue #if docker, make docker-dirs, generate build, run, stop, clear scripts and quit -if (concoretype=="docker"): - for node in nodes_dict: - containername,sourcecode = nodes_dict[node].split(':') - if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 - dockername,langext = sourcecode.split(".") - if not os.path.exists(outdir+"/src/Dockerfile."+dockername): # 3/30/21 - try: +if (concoretype=="docker"): + for node in nodes_dict: + containername,sourcecode = nodes_dict[node].split(':') + if len(sourcecode)!=0 and sourcecode.find(".")!=-1: #3/28/21 + dockername,langext = sourcecode.split(".") + dockerfile_path = os.path.join(outdir, "src", f"Dockerfile.{dockername}") + if not os.path.exists(dockerfile_path): # 3/30/21 + try: if langext=="py": - fsource = open(CONCOREPATH+"/Dockerfile.py") + src_path = CONCOREPATH+"/Dockerfile.py" logging.info("assuming .py extension for Dockerfile") elif langext == "cpp": # 6/22/21 - fsource = open(CONCOREPATH+"/Dockerfile.cpp") + src_path = CONCOREPATH+"/Dockerfile.cpp" logging.info("assuming .cpp extension for Dockerfile") elif langext == "v": # 6/26/21 - fsource = open(CONCOREPATH+"/Dockerfile.v") + src_path = CONCOREPATH+"/Dockerfile.v" logging.info("assuming .v extension for Dockerfile") elif langext == "sh": # 5/19/21 - fsource = open(CONCOREPATH+"/Dockerfile.sh") + src_path = CONCOREPATH+"/Dockerfile.sh" logging.info("assuming .sh extension for Dockerfile") else: - fsource = open(CONCOREPATH+"/Dockerfile.m") + src_path = CONCOREPATH+"/Dockerfile.m" logging.info("assuming .m extension for Dockerfile") - except: - logging.error(f"{CONCOREPATH} is not correct path to concore") - quit() - with open(outdir+"/src/Dockerfile."+dockername,"w") as fcopy: - fcopy.write(fsource.read()) + with open(src_path) as fsource: + source_content = fsource.read() + except: + logging.error(f"{CONCOREPATH} is not correct path to concore") + quit() + dockerfile_parent = os.path.dirname(dockerfile_path) + if dockerfile_parent: + os.makedirs(dockerfile_parent, exist_ok=True) + with open(dockerfile_path,"w") as fcopy: + fcopy.write(source_content) if langext=="py": fcopy.write('CMD ["python", "-i", "'+sourcecode+'"]\n') if langext=="m": @@ -706,7 +745,6 @@ def _resolve_concore_path(): if langext=="v": fcopy.write('RUN iverilog ./'+sourcecode+'\n') # 7/02/21 fcopy.write('CMD ["./a.out"]\n') # 7/02/21 - fsource.close() fbuild.write('#!/bin/bash' + "\n") for node in nodes_dict: @@ -940,16 +978,22 @@ def _resolve_concore_path(): if concoretype=="posix": fbuild.write('#!/bin/bash' + "\n") -for node in nodes_dict: - containername,sourcecode = nodes_dict[node].split(':') - if len(sourcecode)!=0: - if sourcecode.find(".")==-1: - logging.error("cannot pull container "+sourcecode+" with control core type "+concoretype) #3/28/21 - quit() - dockername,langext = sourcecode.split(".") - fbuild.write('mkdir '+containername+"\n") - if concoretype == "windows": - fbuild.write("copy .\\src\\"+sourcecode+" .\\"+containername+"\\"+sourcecode+"\n") +for node in nodes_dict: + containername,sourcecode = nodes_dict[node].split(':') + if len(sourcecode)!=0: + if sourcecode.find(".")==-1: + logging.error("cannot pull container "+sourcecode+" with control core type "+concoretype) #3/28/21 + quit() + dockername,langext = sourcecode.split(".") + fbuild.write('mkdir '+containername+"\n") + source_subdir = os.path.dirname(sourcecode).replace("\\", "/") + if source_subdir: + if concoretype == "windows": + fbuild.write("mkdir .\\"+containername+"\\"+source_subdir.replace("/", "\\")+"\n") + else: + fbuild.write("mkdir -p ./"+containername+"/"+source_subdir+"\n") + if concoretype == "windows": + fbuild.write("copy .\\src\\"+sourcecode+" .\\"+containername+"\\"+sourcecode+"\n") if langext == "py": fbuild.write("copy .\\src\\concore.py .\\" + containername + "\\concore.py\n") elif langext == "cpp": @@ -1237,10 +1281,6 @@ def _resolve_concore_path(): frun.close() fbuild.close() fdebug.close() -fstop.close() -fclear.close() -fmaxtime.close() -fparams.close() if concoretype != "windows": os.chmod(outdir+"/build",stat.S_IRWXU) os.chmod(outdir+"/run",stat.S_IRWXU) @@ -1249,4 +1289,4 @@ def _resolve_concore_path(): os.chmod(outdir+"/clear",stat.S_IRWXU) os.chmod(outdir+"/maxtime",stat.S_IRWXU) os.chmod(outdir+"/params",stat.S_IRWXU) - os.chmod(outdir+"/unlock",stat.S_IRWXU) \ No newline at end of file + os.chmod(outdir+"/unlock",stat.S_IRWXU)