📜Improve markdown support

sorasful · sorasful · commit 85dc362bac36 · 2023-11-10T14:48:37.000+01:00
diff --git a/html_templates/main_template.html b/html_templates/main_template.html
@@ -13,4 +13,4 @@ <h1>Audit Report</h1>
     </div>
     {% endfor %}
 </body>
-</html>
+</html>
diff --git a/html_templates/modules/bandit_report_template.html b/html_templates/modules/bandit_report_template.html
diff --git a/html_templates/modules/gitleaks_report_template.html b/html_templates/modules/gitleaks_report_template.html
diff --git a/html_templates/modules/mypy_report_template.html b/html_templates/modules/mypy_report_template.html
@@ -58,4 +58,4 @@ <h3>Categories with most messages</h3>
             {% endfor %}
         </tbody>
     </table>
-</section>
+</section>
diff --git a/html_templates/modules/radon_report_template.html b/html_templates/modules/radon_report_template.html
diff --git a/html_templates/modules/ruff_report_template.html b/html_templates/modules/ruff_report_template.html
diff --git a/html_templates/modules/structure_report_template.html b/html_templates/modules/structure_report_template.html
diff --git a/html_templates/modules/vulture_report_template.html b/html_templates/modules/vulture_report_template.html
diff --git a/markdown_templates/main_template.md b/markdown_templates/main_template.md
@@ -0,0 +1,11 @@
+# Audit Report
+
+**Project:** {{ project_name }}
+
+**Date:** {{ audit_date }}
+
+{% for name, report_markdown in reports.items() %}
+
+{{ report_markdown }}
+
+{% endfor %}
diff --git a/markdown_templates/modules/bandit_report_template.md b/markdown_templates/modules/bandit_report_template.md
@@ -0,0 +1,39 @@
+## Bandit Security Report
+
+### Report Summary
+Report generated at: {{ data.vulnerabilities.generated_at }}  
+Total number of files analyzed: {{ data.vulnerabilities.metrics | length }}
+
+### Error Details
+{% if data.vulnerabilities.errors %}
+{% for error in data.vulnerabilities.errors %}
+- {{ error }}
+{% endfor %}
+{% else %}
+No errors found.
+{% endif %}
+
+### Metrics Summary
+
+| File | LOC | High Confidence | Medium Confidence | Low Confidence | Undefined Confidence | High Severity | Medium Severity | Low Severity | Undefined Severity | NoSec | Skipped Tests |
+|------|-----|-----------------|-------------------|----------------|----------------------|---------------|-----------------|--------------|--------------------|-------|---------------|
+{% for file_path, metric in data.vulnerabilities.metrics.items() -%}
+| {{ file_path }} | {{ metric.loc }} | {{ metric.CONFIDENCE_HIGH }} | {{ metric.CONFIDENCE_MEDIUM }} | {{ metric.CONFIDENCE_LOW }} | {{ metric.CONFIDENCE_UNDEFINED }} | {{ metric.SEVERITY_HIGH }} | {{ metric.SEVERITY_MEDIUM }} | {{ metric.SEVERITY_LOW }} | {{ metric.SEVERITY_UNDEFINED }} | {{ metric.nosec }} | {{ metric.skipped_tests }} |
+{% endfor -%}
+
+### Detailed Findings
+{% for result in data.vulnerabilities.results %}
+#### Issue in File: {{ result.filename }} at line: {{ result.line_number }}
+**Issue Severity:** {{ result.issue_severity }}  
+**Issue Confidence:** {{ result.issue_confidence }}  
+**CWE ID:** [CWE-{{ result.issue_cwe.id }}]({{ result.issue_cwe.link }})  
+**Issue Description:** {{ result.issue_text }}  
+**Code:**
+
+    {{ result.code | replace('\n', '\n    ') }}
+
+**More Info:** [Click here]({{ result.more_info }})  
+**Test ID:** {{ result.test_id }}  
+**Test Name:** {{ result.test_name }}  
+
+{% endfor %}
diff --git a/markdown_templates/modules/gitleaks_report_template.md b/markdown_templates/modules/gitleaks_report_template.md
@@ -0,0 +1,14 @@
+## Gitleaks Report
+
+### Potential Leaks Detected
+Number of leaks detected: {{ data.leaks | length }}
+
+### Leak Details
+{% for leak in data.leaks %}
+#### File: {{ leak.file }}
+
+| Description | Start Line | End Line | Start Column | End Column | Match | Secret | Commit | Entropy | Author | Email | Date | Message | Rule ID | Fingerprint |
+|-------------|------------|----------|--------------|------------|-------|--------|--------|---------|--------|-------|------|---------|---------|-------------|
+| {{ leak.description }} | {{ leak.start_line }} | {{ leak.end_line }} | {{ leak.start_column }} | {{ leak.end_column }} | {{ leak.match }} | {{ leak.secret }} | {{ leak.commit }} | {{ leak.entropy }} | {{ leak.author }} | {{ leak.email }} | {{ leak.date }} | {{ leak.message }} | {{ leak.rule_id }} | {{ leak.fingerprint }} |
+
+{% endfor %}
diff --git a/markdown_templates/modules/mypy_report_template.md b/markdown_templates/modules/mypy_report_template.md
@@ -0,0 +1,25 @@
+## Mypy Report
+
+### Files with the most messages
+
+| File path | Number of messages |
+|-----------|--------------------|
+{% for file_path, count in data.files_most_messages -%}
+| {{ file_path }} | {{ count }} |
+{% endfor -%}
+
+### Most frequent message
+
+| Message | Number of messages |
+|---------|--------------------|
+{% for msg, count in data.most_messages -%}
+| {{ msg }} | {{ count }} |
+{% endfor -%}
+
+### Categories with most messages
+
+| Category | Number of messages |
+|----------|--------------------|
+{% for category, count in data.most_messages_categories -%}
+| {{ category }} | {{ count }} |
+{% endfor -%}
diff --git a/markdown_templates/modules/radon_report_template.md b/markdown_templates/modules/radon_report_template.md
@@ -0,0 +1,15 @@
+## Radon Complexity Report
+
+### Average Complexity
+Average Complexity of the Project: {{ data.average_complexity }}
+
+### Code Constructs Complexity
+{% for file_path, complexities in data.complexity.items() %}
+#### File: {{ file_path }}
+
+| Name | Type | Complexity | Rank | Line Number | End Line | Column Offset |
+|------|------|------------|------|-------------|----------|---------------|
+{% for item in complexities -%}
+| {{ item.name }} | {{ item.type }} | {{ item.complexity }} | {{ item.rank }} | {{ item.lineno }} | {{ item.endline }} | {{ item.col_offset }} |
+{% endfor -%}
+{% endfor %}
diff --git a/markdown_templates/modules/ruff_report_template.md b/markdown_templates/modules/ruff_report_template.md
@@ -0,0 +1,7 @@
+## Ruff Report
+
+| Code | Message | Fixable | Count |
+|------|---------|---------|-------|
+{% for item in data -%}
+| {{ item.code }} | {{ item.message }} | {{ item.fixable }} | {{ item.count }} |
+{% endfor -%}
diff --git a/markdown_templates/modules/structure_report_template.md b/markdown_templates/modules/structure_report_template.md
@@ -0,0 +1,47 @@
+## Structure Report
+
+### Git Information
+Is Git Repository: {{ data.is_git_repository }}
+{% if data.git_repository_hash %}
+Commit Hash: {{ data.git_repository_hash }}
+{% endif %}
+
+### Project Structure
+{% macro render_directory(item, prefix='') -%}
+{{ prefix }}- {{ item.name }}
+  {%- if item.type == "directory" and item.contents %}
+    {%- for subitem in item.contents %}
+      {{ render_directory(subitem, prefix + '    ') }}
+    {%- endfor %}
+  {%- endif %}
+{%- endmacro %}
+
+```
+{% for item in data.structure %}
+{{ render_directory(item) }}
+{% endfor %}
+```
+
+### Configuration Files
+Has Precommit File: {{ data.has_precommit_file }}
+{% if data.has_precommit_file %}
+#### Pre-commit hooks
+{% for repo in data.precommit_config.repos %}
+- {{ repo.repo }}
+  {%- for hook in repo.hooks %}
+    - {{ hook.id }} : {{ hook.name | default(hook.id) }}
+  {%- endfor %}
+{% endfor %}
+{% endif %}
+
+### Testing
+Has Tests: {{ data.has_tests }}  
+Uses Pytest: {{ data.uses_pytest }}  
+{% if data.uses_pytest and data.coverage %}
+Coverage: {{ data.coverage['totals']['percent_covered_display'] }}%
+{% endif %}
+
+### Dependencies Detection
+{% for dep, exists in data.dependencies.items() %}
+- {{ dep }}: {{ exists }}
+{% endfor %}
diff --git a/markdown_templates/modules/vulture_report_template.md b/markdown_templates/modules/vulture_report_template.md
@@ -0,0 +1,7 @@
+## Vulture Report
+
+| File path | Line | Message | Confidence |
+|-----------|------|---------|------------|
+{% for item in data -%}
+| {{ item.file_path }} | {{ item.line }} | {{ item.msg }} | {{ item.confidence }} |
+{% endfor -%}
diff --git a/requirements.txt b/requirements.txt
@@ -8,8 +8,8 @@ attrs==23.1.0
     # via
     #   jsonschema
     #   referencing
-beautifulsoup4==4.12.2
-    # via markdownify
+bandit==1.7.5
+    # via unella (setup.py)
 click==8.1.7
     # via wily
 colorama==0.4.6
@@ -23,7 +23,9 @@ future==0.18.3
 gitdb==4.0.11
     # via gitpython
 gitpython==3.1.40
-    # via wily
+    # via
+    #   bandit
+    #   wily
 jinja2==3.1.2
     # via unella (setup.py)
 jsonschema==4.19.2
@@ -38,8 +40,6 @@ mando==0.6.4
     # via radon
 markdown-it-py==3.0.0
     # via rich
-markdownify==0.11.6
-    # via unella (setup.py)
 markupsafe==2.1.3
     # via jinja2
 mdurl==0.1.2
@@ -52,6 +52,8 @@ nbformat==5.9.2
     # via wily
 packaging==23.2
     # via plotly
+pbr==6.0.0
+    # via stevedore
 platformdirs==3.11.0
     # via jupyter-core
 plotly==5.18.0
@@ -61,7 +63,9 @@ progress==1.6
 pygments==2.16.1
     # via rich
 pyyaml==6.0.1
-    # via unella (setup.py)
+    # via
+    #   bandit
+    #   unella (setup.py)
 radon==5.1.0
     # via
     #   unella (setup.py)
@@ -71,21 +75,21 @@ referencing==0.30.2
     #   jsonschema
     #   jsonschema-specifications
 rich==13.6.0
-    # via unella (setup.py)
+    # via
+    #   bandit
+    #   unella (setup.py)
 rpds-py==0.10.6
     # via
     #   jsonschema
     #   referencing
 ruff==0.1.3
     # via unella (setup.py)
 six==1.16.0
-    # via
-    #   mando
-    #   markdownify
+    # via mando
 smmap==5.0.1
     # via gitdb
-soupsieve==2.5
-    # via beautifulsoup4
+stevedore==5.1.0
+    # via bandit
 tabulate==0.9.0
     # via wily
 tenacity==8.2.3
diff --git a/setup.cfg b/setup.cfg
@@ -28,7 +28,6 @@ install_requires =
     loguru>=0.7.2
     jinja2>=3.1.2
     PyYAML>=6.0.1
-    markdownify>=0.11.6
 python_requires = >=3.10.1
 
 [options.packages.find]
diff --git a/tests/modules/test_bandit.py b/tests/modules/test_bandit.py
@@ -9,18 +9,19 @@ def test_bandit_report() -> None:
 
     first_vulnerability, *vulnerabilities = results["vulnerabilities"]["results"]
 
-    assert first_vulnerability == {
-        "code": '7 \n8 response = requests.get("https://pro.pinock.io/features")\n9 \n',
-        "col_offset": 11,
-        "end_col_offset": 57,
-        "filename": "/home/tevak/dev/unella/pan.py",
-        "issue_confidence": "LOW",
-        "issue_cwe": {"id": 400, "link": "https://cwe.mitre.org/data/definitions/400.html"},
-        "issue_severity": "MEDIUM",
-        "issue_text": "Requests call without timeout",
-        "line_number": 8,
-        "line_range": [8],
-        "more_info": "https://bandit.readthedocs.io/en/1.7.5/plugins/b113_request_without_timeout.html",
-        "test_id": "B113",
-        "test_name": "request_without_timeout",
-    }
+    assert isinstance(first_vulnerability.get("code"), str)
+    assert isinstance(first_vulnerability.get("col_offset"), int)
+    assert isinstance(first_vulnerability.get("end_col_offset"), int)
+    assert isinstance(first_vulnerability.get("filename"), str)
+    assert isinstance(first_vulnerability.get("issue_confidence"), str)
+    assert isinstance(first_vulnerability.get("issue_cwe"), dict)
+    assert isinstance(first_vulnerability["issue_cwe"].get("id"), int)
+    assert isinstance(first_vulnerability["issue_cwe"].get("link"), str)
+    assert isinstance(first_vulnerability.get("issue_severity"), str)
+    assert isinstance(first_vulnerability.get("issue_text"), str)
+    assert isinstance(first_vulnerability.get("line_number"), int)
+    assert isinstance(first_vulnerability.get("line_range"), list)
+    assert all(isinstance(item, int) for item in first_vulnerability.get("line_range"))
+    assert isinstance(first_vulnerability.get("more_info"), str)
+    assert isinstance(first_vulnerability.get("test_id"), str)
+    assert isinstance(first_vulnerability.get("test_name"), str)
diff --git a/tests/modules/test_structure.py b/tests/modules/test_structure.py
@@ -32,7 +32,7 @@ def test_structure_report(get_coverage_mock) -> None:
                 "rev": "v2.4.0",
                 "hooks": [
                     {"id": "check-ast"},
-                    {"id": "trailing-whitespace"},
+                    {"id": "trailing-whitespace", "args": ["--markdown-linebreak-ext=md"]},
                     {"id": "check-toml"},
                     {"id": "end-of-file-fixer"},
                 ],
diff --git a/unella/audit_generator.py b/unella/audit_generator.py
@@ -6,7 +6,6 @@
 
 from jinja2 import Environment, FileSystemLoader
 from loguru import logger
-from markdownify import markdownify as md
 
 from unella.cli import ProgressBar
 from unella.modules.bandit.main import BanditReport
@@ -71,26 +70,33 @@ def get_data(self) -> dict[str, Any]:
     def get_json(self) -> str:
         return json.dumps(self.get_data())
 
-    def get_html(self) -> str:
-        env = Environment(loader=FileSystemLoader("templates"))
-        template = env.get_template("main_template.html")
+    def get_report(self, format: str = "html") -> str:
+        if format == "html":
+            env = Environment(loader=FileSystemLoader("html_templates"))
+            template = env.get_template("main_template.html")
+        else:
+            env = Environment(loader=FileSystemLoader("markdown_templates"))
+            template = env.get_template("main_template.md")
 
-        reports_html = {}
+        reports = {}
         available_report_list = list(self.available_report_list)
         progress_bar = ProgressBar(len(available_report_list))
         for step, report_class in zip(progress_bar.start(), available_report_list):
             report_name = pascal_case_to_snake_case(report_class.__name__)
             try:
-                reports_html[report_name] = report_class(str(self.project_path)).to_html()
+                report = report_class(str(self.project_path))
+                reports[report_name] = report.to_html() if format == "html" else report.to_markdown()
             except Exception as e:
                 logger.exception(f"Got an exception on module {report_name}, skipping this one")
                 continue
         now = datetime.datetime.now()
         formatted_now = now.strftime("%B %d, %Y %H:%M:%S")
         project_name = self.project_path.name
 
-        return template.render(reports=reports_html, audit_date=formatted_now, project_name=project_name)
+        return template.render(reports=reports, audit_date=formatted_now, project_name=project_name)
+
+    def get_html(self) -> str:
+        return self.get_report(format="html")
 
     def get_markdown(self) -> str:
-        html = self.get_html()
-        return md(html)
+        return self.get_report(format="markdown")
diff --git a/unella/modules/generic.py b/unella/modules/generic.py
@@ -33,7 +33,13 @@ def get_results(self):
         raise NotImplementedError
 
     def to_html(self) -> str:
-        env = Environment(loader=FileSystemLoader("templates/modules"))
+        env = Environment(loader=FileSystemLoader("html_templates/modules"))
         template_name = f"{pascal_case_to_snake_case(self.__class__.__name__)}_template.html"
         template = env.get_template(template_name)
         return template.render(data=self.get_results())
+
+    def to_markdown(self) -> str:
+        env = Environment(loader=FileSystemLoader("markdown_templates/modules"))
+        template_name = f"{pascal_case_to_snake_case(self.__class__.__name__)}_template.md"
+        template = env.get_template(template_name)
+        return template.render(data=self.get_results())

-Original file line number
+Diff line change
     </div>
     {% endfor %}
 </body>
 -</html>
 +</html>