Merge pull request #51 from rmusser01/dev

Sync

Author: rmusser01

Tests/UI/test_ingest_window.py

@@ -0,0 +1,225 @@
+# test_ingest_window.py
+#
+# Imports
+import pytest
+from pytest_mock import MockerFixture  # For mocking
+from pathlib import Path
+#
+# Third-party Libraries
+from textual.app import App, ComposeResult
+from textual.widgets import Button, Input, Select, Checkbox, TextArea, RadioSet, RadioButton, Collapsible, ListView, \
+    ListItem, Markdown, LoadingIndicator, Label, Static
+from textual.containers import Container, VerticalScroll, Horizontal, Vertical
+from textual.pilot import Pilot
+from textual.css.query import QueryError
+#
+# Local Imports
+from tldw_chatbook.app import TldwCli  # The main app
+from tldw_chatbook.UI.Ingest_Window import IngestWindow, MEDIA_TYPES  # Import MEDIA_TYPES
+from tldw_chatbook.tldw_api.schemas import ProcessVideoRequest, ProcessAudioRequest, ProcessPDFRequest, \
+    ProcessEbookRequest, ProcessDocumentRequest, ProcessXMLRequest, ProcessMediaWikiRequest
+#
+#
+########################################################################################################################
+#
+# Fixtures and Helper Functions
+
+# Helper to get the IngestWindow instance from the app
+async def get_ingest_window(pilot: Pilot) -> IngestWindow:
+    ingest_window_query = pilot.app.query(IngestWindow)
+    assert ingest_window_query.is_empty is False, "IngestWindow not found"
+    return ingest_window_query.first()
+
+
+@pytest.fixture
+async def app_pilot() -> Pilot:
+    app = TldwCli()
+    async with app.run_test() as pilot:
+        # Ensure the Ingest tab is active. Default is Chat.
+        # Switching tabs is handled by app.py's on_button_pressed for tab buttons.
+        # We need to find the Ingest tab button and click it.
+        # Assuming tab IDs are like "tab-ingest"
+        try:
+            await pilot.click("#tab-ingest")
+        except QueryError:
+            # Fallback if direct ID click isn't working as expected in test setup
+            # This might indicate an issue with tab IDs or pilot interaction timing
+            all_buttons = pilot.app.query(Button)
+            ingest_tab_button = None
+            for btn in all_buttons:
+                if btn.id == "tab-ingest":
+                    ingest_tab_button = btn
+                    break
+            assert ingest_tab_button is not None, "Ingest tab button not found"
+            await pilot.click(ingest_tab_button)
+
+        # Verify IngestWindow is present and active
+        ingest_window = await get_ingest_window(pilot)
+        assert ingest_window is not None
+        assert ingest_window.display is True, "IngestWindow is not visible after switching to Ingest tab"
+        # Also check the app's current_tab reactive variable
+        assert pilot.app.current_tab == "ingest", "App's current_tab is not set to 'ingest'"
+        yield pilot
+
+
+# Test Class
+class TestIngestWindowTLDWAPI:
+
+    async def test_initial_tldw_api_nav_buttons_and_views(self, app_pilot: Pilot):
+        ingest_window = await get_ingest_window(app_pilot)
+        # The IngestWindow itself is a container, nav buttons are direct children of its "ingest-nav-pane"
+        nav_pane = ingest_window.query_one("#ingest-nav-pane")
+
+        for mt in MEDIA_TYPES:
+            nav_button_id = f"ingest-nav-tldw-api-{mt.replace('_', '-')}"  # IDs don't have #
+            view_id = f"ingest-view-tldw-api-{mt.replace('_', '-')}"
+
+            # Check navigation button exists
+            nav_button = nav_pane.query_one(f"#{nav_button_id}", Button)
+            assert nav_button is not None, f"Navigation button {nav_button_id} not found"
+            expected_label_part = mt.replace('_', ' ').title()
+            if mt == "mediawiki_dump":
+                expected_label_part = "MediaWiki Dump"
+            assert expected_label_part in str(nav_button.label), f"Label for {nav_button_id} incorrect"
+
+            # Check view area exists
+            view_area = ingest_window.query_one(f"#{view_id}", Container)
+            assert view_area is not None, f"View area {view_id} not found"
+
+            # Check initial visibility based on app's active ingest view
+            # This assumes that after switching to Ingest tab, a default sub-view *within* Ingest is activated.
+            # If `ingest_active_view` is set (e.g. to "ingest-view-prompts" by default), then
+            # all tldw-api views should be hidden.
+            active_ingest_view_on_app = app_pilot.app.ingest_active_view
+            if view_id != active_ingest_view_on_app:
+                assert view_area.display is False, f"{view_id} should be hidden if not the active ingest view ('{active_ingest_view_on_app}')"
+            else:
+                assert view_area.display is True, f"{view_id} should be visible as it's the active ingest view ('{active_ingest_view_on_app}')"
+
+    @pytest.mark.parametrize("media_type", MEDIA_TYPES)
+    async def test_tldw_api_navigation_and_view_display(self, app_pilot: Pilot, media_type: str):
+        ingest_window = await get_ingest_window(app_pilot)
+        nav_button_id = f"ingest-nav-tldw-api-{media_type.replace('_', '-')}"
+        target_view_id = f"ingest-view-tldw-api-{media_type.replace('_', '-')}"
+
+        await app_pilot.click(f"#{nav_button_id}")
+        await app_pilot.pause()  # Allow watchers to update display properties
+
+        # Verify target view is visible
+        target_view_area = ingest_window.query_one(f"#{target_view_id}", Container)
+        assert target_view_area.display is True, f"{target_view_id} should be visible after clicking {nav_button_id}"
+        assert app_pilot.app.ingest_active_view == target_view_id, f"App's active ingest view should be {target_view_id}"
+
+        # Verify other TLDW API views are hidden
+        for other_mt in MEDIA_TYPES:
+            if other_mt != media_type:
+                other_view_id = f"ingest-view-tldw-api-{other_mt.replace('_', '-')}"
+                other_view_area = ingest_window.query_one(f"#{other_view_id}", Container)
+                assert other_view_area.display is False, f"{other_view_id} should be hidden when {target_view_id} is active"
+
+        # Verify common form elements exist with dynamic IDs
+        common_endpoint_input = target_view_area.query_one(f"#tldw-api-endpoint-url-{media_type}", Input)
+        assert common_endpoint_input is not None
+
+        common_submit_button = target_view_area.query_one(f"#tldw-api-submit-{media_type}", Button)
+        assert common_submit_button is not None
+
+        # Verify media-specific options container and its widgets
+        if media_type == "video":
+            opts_container = target_view_area.query_one("#tldw-api-video-options", Container)
+            assert opts_container.display is True
+            widget = opts_container.query_one(f"#tldw-api-video-transcription-model-{media_type}", Input)
+            assert widget is not None
+        elif media_type == "audio":
+            opts_container = target_view_area.query_one("#tldw-api-audio-options", Container)
+            assert opts_container.display is True
+            widget = opts_container.query_one(f"#tldw-api-audio-transcription-model-{media_type}", Input)
+            assert widget is not None
+        elif media_type == "pdf":
+            opts_container = target_view_area.query_one("#tldw-api-pdf-options", Container)
+            assert opts_container.display is True
+            widget = opts_container.query_one(f"#tldw-api-pdf-engine-{media_type}", Select)
+            assert widget is not None
+        elif media_type == "ebook":
+            opts_container = target_view_area.query_one("#tldw-api-ebook-options", Container)
+            assert opts_container.display is True
+            widget = opts_container.query_one(f"#tldw-api-ebook-extraction-method-{media_type}", Select)
+            assert widget is not None
+        elif media_type == "document":  # Has minimal specific options currently
+            opts_container = target_view_area.query_one("#tldw-api-document-options", Container)
+            assert opts_container.display is True
+            # Example: find the label if one exists
+            try:
+                label = opts_container.query_one(Label)  # Assuming there's at least one label
+                assert label is not None
+            except QueryError:  # If no labels, this is fine for doc
+                pass
+        elif media_type == "xml":
+            opts_container = target_view_area.query_one("#tldw-api-xml-options", Container)
+            assert opts_container.display is True
+            widget = opts_container.query_one(f"#tldw-api-xml-auto-summarize-{media_type}", Checkbox)
+            assert widget is not None
+        elif media_type == "mediawiki_dump":
+            opts_container = target_view_area.query_one("#tldw-api-mediawiki-options", Container)
+            assert opts_container.display is True
+            widget = opts_container.query_one(f"#tldw-api-mediawiki-wiki-name-{media_type}", Input)
+            assert widget is not None
+
+    async def test_tldw_api_video_submission_data_collection(self, app_pilot: Pilot, mocker: MockerFixture):
+        media_type = "video"
+        ingest_window = await get_ingest_window(app_pilot)
+
+        # Navigate to video tab by clicking its nav button
+        nav_button_id = f"ingest-nav-tldw-api-{media_type}"
+        await app_pilot.click(f"#{nav_button_id}")
+        await app_pilot.pause()  # Allow UI to update
+
+        target_view_id = f"ingest-view-tldw-api-{media_type}"
+        target_view_area = ingest_window.query_one(f"#{target_view_id}", Container)
+        assert target_view_area.display is True, "Video view area not displayed after click"
+
+        # Mock the API client and its methods
+        mock_api_client_instance = mocker.MagicMock()
+        # Make process_video an async mock
+        mock_process_video = mocker.AsyncMock(return_value=mocker.MagicMock())
+        mock_api_client_instance.process_video = mock_process_video
+        mock_api_client_instance.close = mocker.AsyncMock()
+
+        mocker.patch("tldw_chatbook.Event_Handlers.ingest_events.TLDWAPIClient", return_value=mock_api_client_instance)
+
+        # Set form values
+        endpoint_url_input = target_view_area.query_one(f"#tldw-api-endpoint-url-{media_type}", Input)
+        urls_textarea = target_view_area.query_one(f"#tldw-api-urls-{media_type}", TextArea)
+        video_trans_model_input = target_view_area.query_one(f"#tldw-api-video-transcription-model-{media_type}", Input)
+        auth_method_select = target_view_area.query_one(f"#tldw-api-auth-method-{media_type}", Select)
+
+        endpoint_url_input.value = "http://fakeapi.com"
+        urls_textarea.text = "http://example.com/video.mp4"
+        video_trans_model_input.value = "test_video_model"
+        auth_method_select.value = "config_token"
+
+        app_pilot.app.app_config = {"tldw_api": {"auth_token_config": "fake_token"}}
+
+        submit_button_id = f"tldw-api-submit-{media_type}"
+        await app_pilot.click(f"#{submit_button_id}")
+        await app_pilot.pause(delay=0.5)
+
+        mock_process_video.assert_called_once()
+        call_args = mock_process_video.call_args[0]
+
+        assert len(call_args) >= 1, "process_video not called with request_model"
+        request_model_arg = call_args[0]
+
+        assert isinstance(request_model_arg, ProcessVideoRequest)
+        assert request_model_arg.urls == ["http://example.com/video.mp4"]
+        assert request_model_arg.transcription_model == "test_video_model"
+        assert request_model_arg.api_key == "fake_token"
+
+        # Example for local_file_paths if it's the second argument
+        if len(call_args) > 1:
+            local_files_arg = call_args[1]
+            assert local_files_arg == [], "local_files_arg was not empty"
+        else:
+            # This case implies process_video might not have received local_file_paths,
+            # which could be an issue if it's expected. For now, let's assume it's optional.
+            pass

pyproject.toml

@@ -4,27 +4,26 @@ requires = ["setuptools>=61.0"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "tldw_chatbook" # This is the distribution name (what you pip install)
+name = "tldw_chatbook"
 version = "0.1.0"
 authors = [
   { name="Robert Musser", email="[email protected]" },
 ]
 description = "A Textual TUI for chatting with LLMs, and interacting with the tldw server."
-readme = "README.md" # Assumes README.md is in the project root
+readme = "README.md"
 requires-python = ">=3.11"
-license = { file="LICENSE" } # Assumes LICENSE file is in the project root
+license = { file="LICENSE" }
 keywords = ["tui", "cli", "llm", "textual", "ai", "chat"]
 classifiers = [
     "Development Status :: 3 - Alpha",
     "Intended Audience :: Developers",
     "Intended Audience :: End Users/Desktop",
     "Topic :: Utilities",
     "Topic :: Terminals",
-    "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", # Corrected to AGPL
+    "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
-    # "Programming Language :: Python :: 3.13", # Only list if you actively test/support it
     "Environment :: Console",
     "Operating System :: OS Independent",
 ]
@@ -34,9 +33,9 @@ dependencies = [
     "chardet",
     "httpx",
     "loguru",
-    "textual>=3.3.0", # Specify a minimum Textual version if features depend on it
+    "textual>=3.3.0",
     "requests",
-     "rich", # Usually pulled in by Textual, but explicit is fine
+     "rich",
     "Pillow",
     "PyYAML",
     "pydantic",

requirements.txt

@@ -10,7 +10,7 @@ loguru
 pydantic
 pyyaml
 httpx
-pytest
+#pytest
 emoji
 # Chunk Library
 #tqdm

tldw_chatbook/DB/Client_Media_DB_v2.py

@@ -1917,9 +1917,12 @@ def add_media_with_keywords(self,
                     # --- Handle Unvectorized Chunks ---
                     if chunks is not None:  # chunks argument was provided (could be empty or list of dicts)
                         if action == "updated":
-                            # Hard delete old chunks for this media_id if updating
-                            logging.debug(f"Hard deleting existing UnvectorizedMediaChunks for updated media_id {media_id}.")
-                            conn.execute("DELETE FROM UnvectorizedMediaChunks WHERE media_id = ?", (media_id,))
+                            # If overwriting and new chunks are provided, clear old ones.
+                            # If `chunks` is an empty list, it also means clear old ones.
+                            if overwrite:  # Only delete if overwrite is true
+                                logging.debug(
+                                    f"Hard deleting existing UnvectorizedMediaChunks for updated media_id {media_id} due to overwrite and new chunks being provided.")
+                                conn.execute("DELETE FROM UnvectorizedMediaChunks WHERE media_id = ?", (media_id,))
 
                         num_chunks_saved = 0
                         if chunks:  # If chunks list is not empty
@@ -1980,9 +1983,18 @@ def add_media_with_keywords(self,
                                     raise DatabaseError(f"Failed to save chunk {i} due to integrity constraint: {e}") from e
                             logging.info(f"Saved {num_chunks_saved} unvectorized chunks for media_id {media_id}.")
 
-                        # Update Media chunking_status to 'completed' as chunk processing is done (even if 0 chunks were provided)
-                        conn.execute("UPDATE Media SET chunking_status = 'completed' WHERE id = ?", (media_id,))
-                        logging.debug(f"Updated Media chunking_status to 'completed' for media_id {media_id} after chunk processing.")
+                            # Update Media chunking_status
+                            # If chunks were provided (even an empty list, meaning "clear existing and add these (none)"),
+                            # then chunking is considered 'completed' from the perspective of this operation.
+                            # If `chunks` was None (meaning "don't touch existing chunks"), status remains as is or 'pending'.
+                            final_chunking_status_for_media = 'completed'  # if chunks is not None
+                            # If the main `perform_chunking` flag (from request, not DB field) was false,
+                            # then perhaps status should be different. For now, if chunks data is passed, it's 'completed'.
+                            # This might need more nuanced logic based on the `perform_chunking` flag from the original request.
+                            conn.execute("UPDATE Media SET chunking_status = ? WHERE id = ?",
+                                         (final_chunking_status_for_media, media_id,))
+                            logging.debug(
+                                f"Updated Media chunking_status to '{final_chunking_status_for_media}' for media_id {media_id} after chunk processing.")
 
                     # Original chunk_options placeholder log
                     if chunk_options: