Cache users aggressively

This commit was automatically generated by a script: https://gitfub.space/Jmaa/repo-manager

.gitea/workflows/python-package.yml

@@ -10,7 +10,7 @@ jobs:
     uses: jmaa/workflows/.gitea/workflows/python-package.yaml@v6.21
     with:
       REGISTRY_DOMAIN: gitfub.space
-      REGISTRY_ORGANIZATION: usagi-keiretsu
+      REGISTRY_ORGANIZATION: jmaa
     secrets:
       PIPY_REPO_USER: ${{ secrets.PIPY_REPO_USER }}
       PIPY_REPO_PASS: ${{ secrets.PIPY_REPO_PASS }}

.gitea/workflows/python-test.yml

@@ -1,4 +1,4 @@
-name: Test Python
+name: Run Python tests (through Pytest)
 
 on:
   push:
@@ -6,4 +6,26 @@ on:
 
 jobs:
   Test:
-    uses: jmaa/workflows/.gitea/workflows/python-test.yaml@v6.21
+    runs-on: ubuntu-latest
+    container:
+      image: node:21-bookworm
+    steps:
+      - name: Setting up Python ${{ env.PYTHON_VERSION }} for ${{runner.arch}} ${{runner.os}}
+        run: |
+           apt-get update
+           apt-get install -y python3 python3-pip
+      - name: Check out repository code
+        if: success()
+        uses: actions/checkout@v3
+      - name: Installing Python Dependencies
+        if: success()
+        run: python3 -m pip install --upgrade pip setuptools wheel build twine pytest pytest-cov --break-system-packages
+      - name: Installing Python Test Dependencies
+        if: success() && hashFiles('requirements_test.txt') != ''
+        run: python3 -m pip install --upgrade -r requirements_test.txt --break-system-packages
+      - name: Installing package
+        if: success()
+        run: python3 -m pip install .[test]  --break-system-packages
+      - name: Test Python Code
+        if: success()
+        run: python3 -m pytest test --cov=favro_sync --cov-report html:htmlcov --cov-fail-under=0

.gitea/workflows/python-version-check.yml

@@ -0,0 +1,28 @@
+name: Verify Python project can be installed, loaded and have version checked
+
+on:
+  push:
+    paths-ignore: ["README.md", ".gitignore", "LICENSE", "ruff.toml"]
+
+jobs:
+  Test:
+    runs-on: ubuntu-latest
+    container:
+      image: node:21-bookworm
+    steps:
+      - name: Setting up Python ${{ env.PYTHON_VERSION }} for ${{runner.arch}} ${{runner.os}}
+        run: |
+           apt-get update
+           apt-get install -y python3 python3-pip
+      - name: Check out repository code
+        if: success()
+        uses: actions/checkout@v3
+      - name: Installing Python Dependencies
+        if: success()
+        run: python3 -m pip install --upgrade pip setuptools wheel build twine --break-system-packages
+      - name: Installing package
+        if: success()
+        run: python3 -m pip install .  --break-system-packages
+      - name: Check version field
+        if: success()
+        run: python3 -c "import favro_sync; assert favro_sync.__version__ is not None"

README.md

@@ -4,7 +4,9 @@
 
 
 
-# Favro Sync.
+# Favro Sync
+
+![Test program/library](https://gitfub.space/Jmaa/favro-sync/actions/workflows/python-test.yml/badge.svg)
 
 Filesystem in User Space for Favro.
 
@@ -24,6 +26,7 @@ Features:
   - Tags
   - Assignees
   - Dependencies
+  - Custom fields
 - Change card features:
   - Title
   - Description
@@ -59,14 +62,22 @@ Limitations:
 
 Following features are work in progress:
 
-- [ ] Frontmatter: Update Tags
-- [ ] Frontmatter: Updated assigned members
-- [ ] Frontmatter: Arbitrary structured data? Read-only.
-- [ ] Frontmatter: Dependencies. As vault links in Obsidian mode.
-- [ ] Allow users to toggle Obsidian mode, instead of being default.
-- [ ] Get the correct last-modified date.
-- [ ] Improve cache behaviour. User and tags can have much longer cache times.
-
+- [ ] Frontmatter: Writable Tags
+- [ ] Frontmatter: Writable assigned members
+- [ ] Frontmatter: Writable Tasks.
+    1. Save updated TaskList along with card (using `PUT cards`)
+    2. Get the Card's TaskList.
+    3. Remove all TaskList's except for the latest (how to determine latest?)
+    4. That's three requests just to save a freaking list of tasks!
+- [ ] Frontmatter: Writable Dependencies.
+- [ ] Usability: Richer directory structure
+- [ ] Usability: Allow users to toggle Obsidian mode, instead of being default.
+- [ ] Precision: Get the correct last-modified date.
+- [ ] Performance: Improve cache behaviour. User and tags can have much longer cache times.
+- [ ] Performance: Parallelize requests.
+    * Paginated pages can be easily parallelize.
+- [X] Frontmatter: Arbitrary structured data (Custom Fields)? Read-only.
+- [X] Frontmatter: Readable Dependencies. As vault links in Obsidian mode.
 
 ## Dependencies
 
@@ -81,6 +92,7 @@ Full list of requirements:
 - [requests-cache](https://pypi.org/project/requests-cache/)
 - [fuse-python](https://pypi.org/project/fuse-python/)
 - [secret_loader](https://gitfub.space/Jmaa/secret_loader)
+- [requests_util](https://gitfub.space/Jmaa/requests_util)
 - [marko](https://pypi.org/project/marko/)
 - [python-frontmatter](https://pypi.org/project/python-frontmatter/)
 

favro_sync/__init__.py

@@ -43,6 +43,8 @@ Limitations:
    [`python-fuse`](https://github.com/libfuse/python-fuse) implements a whole
    bunch automatically.)
 
+Use `umount` to unmount the cards again.
+
 ## Architecture
 
 - `FavroFuse`

favro_sync/__main__.py

@@ -10,6 +10,7 @@ from .favro_fuse import start_favro_fuse
 
 def main():
     logging.basicConfig()
+    logging.getLogger().setLevel('INFO')
 
     read_only = False
 

favro_sync/favro_client.py

@@ -4,6 +4,8 @@ Implements methods for interacting with the  [Favro API](https://favro.com/devel
 """
 
 import dataclasses
+from typing import Any
+import requests_cache
 import datetime
 from collections.abc import Iterator
 from logging import getLogger
@@ -33,6 +35,7 @@ logger = getLogger(__name__)
 URL_API_ROOT = 'https://favro.com/api/v1'
 URL_GET_CARDS = URL_API_ROOT + '/cards'
 URL_UPDATE_CARD = URL_API_ROOT + '/cards/{card_id}'
+URL_GET_USERS = URL_API_ROOT + '/users/'
 URL_GET_USER = URL_API_ROOT + '/users/{user_id}'
 URL_GET_TAG = URL_API_ROOT + '/tags/{tag_id}'
 URL_GET_CUSTOM_FIELD = URL_API_ROOT + '/customfields/{custom_field_id}'
@@ -105,6 +108,7 @@ class FavroClient:
         self.read_only = read_only
 
         self.card_cache = CardCache()
+        self.user_cache = None
 
         # Setup caching
         requests_util.setup_limiter(
@@ -116,6 +120,9 @@ class FavroClient:
         requests_util.setup_limiter(
             self.session, URL_GET_TASKS, datetime.timedelta(minutes=10),
         )
+        requests_util.setup_limiter(
+            self.session, URL_GET_CUSTOM_FIELD, datetime.timedelta(days=30),
+        )
 
     def check_logged_in(self) -> None:
         next(self.get_todo_list_cards())
@@ -130,6 +137,7 @@ class FavroClient:
         collection_id: CollectionId | None = None,
         todo_list=False,
     ) -> Iterator[Card]:
+        logger.info('Getting cards: seq_id=%s, collection_id=%s, todo_list=%s', seq_id, collection_id, todo_list)
         request = self._get_cards_request(
             seq_id=seq_id, todo_list=todo_list, collection_id=collection_id,
         )
@@ -165,27 +173,44 @@ class FavroClient:
         return requests.Request('GET', URL_GET_CARDS, params=params)
 
     def get_card(self, seq_id: SeqId) -> Card:
-        if card := self.card_cache.get_card_by_seq_id(seq_id):
+        if card := self.get_card_if_cached(seq_id):
             return card
         return next(self.get_cards(seq_id=seq_id))
 
+    def get_card_if_cached(self, seq_id: SeqId) -> Card | None:
+        if card := self.card_cache.get_card_by_seq_id(seq_id):
+            return card
+        return None
+
     def get_card_by_card_id(self, card_id: CardId) -> Card:
-        response = self.session.get(URL_UPDATE_CARD.format(card_id=card_id.raw_id))
-        return Card.from_json(response.json())
+        json_data = self._get_json(URL_UPDATE_CARD.format(card_id=card_id.raw_id))
+        return Card.from_json(json_data)
 
     def get_user(self, user_id: UserId) -> UserInfo:
-        response = self.session.get(URL_GET_USER.format(user_id=user_id.raw_id))
-        return UserInfo.from_json(response.json())
+        self._load_users()
+        return self.user_cache[user_id]
+
+    def _load_users(self) -> None:
+        if self.user_cache is not None:
+            return
+
+        request = requests.Request('GET', URL_GET_USERS)
+
+        user_cache = {}
+        for user_info in self._get_paginated(request, UserInfo.from_json):
+            user_cache[user_info.user_id] = user_info
+        self.user_cache = user_cache
+        logger.info('Loaded %s users', len(user_cache))
 
     def get_tag(self, tag_id: TagId) -> TagInfo:
-        response = self.session.get(URL_GET_TAG.format(tag_id=tag_id.raw_id))
-        return TagInfo.from_json(response.json())
+        json_data = self._get_json(URL_GET_TAG.format(tag_id=tag_id.raw_id))
+        return TagInfo.from_json(json_data)
 
     def get_custom_field(self, custom_field_id: CustomFieldId) -> CustomFieldInfo:
-        response = self.session.get(
+        json_data = self._get_json(
             URL_GET_CUSTOM_FIELD.format(custom_field_id=custom_field_id.raw_id),
         )
-        return CustomFieldInfo.from_json(response.json())
+        return CustomFieldInfo.from_json(json_data)
 
     def _invalidate_cache(self, card_id: CardId) -> None:
         card = self.card_cache.remove(card_id)
@@ -234,7 +259,7 @@ class FavroClient:
         }
 
         url = URL_UPDATE_CARD.format(card_id=card_id.raw_id)
-        logger.warning('Sending request: %s', url)
+        logger.info('Sending PUT: %s', url)
         response = self.session.put(
             url,
             json=json_body,
@@ -257,9 +282,11 @@ class FavroClient:
             request = self.session.prepare_request(base_request)
 
             # Run query
-            logger.warning('Sending request: %s', request.url)
+            logger.debug('Sending GET: %s', request.url)
             response = self.session.send(request)
             response.raise_for_status()
+            if not response.from_cache:
+                logger.warning('Got new result: %s', request.url)
             json = response.json()
 
             for entity_json in json['entities']:
@@ -271,3 +298,9 @@ class FavroClient:
             page = json['page'] + 1
             request_id = json['requestId']
             num_pages = json['pages']
+
+    def _get_json(self, url: str) -> dict[str, Any]:
+        logger.debug('Sending GET: %s', url)
+        response = self.session.get(url)
+        response.raise_for_status()
+        return response.json()

favro_sync/favro_fuse.py

@@ -58,6 +58,7 @@ def to_custom_fields(card: Card, favro_client: FavroClient) -> dict[str, str]:
 
 
 def to_card_contents(card: Card, favro_client: FavroClient) -> CardContents:
+    logger.info('Getting card contents for card: PAR-%s', card.seq_id)
     tags = [favro_client.get_tag(tag_id).name for tag_id in card.tags]
     assignments = [
         favro_client.get_user(assignment.user).name for assignment in card.assignments
@@ -69,6 +70,8 @@ def to_card_contents(card: Card, favro_client: FavroClient) -> CardContents:
         for dep in card.dependencies
         if dep.is_before
     ]
+    custom_fields = to_custom_fields(card, favro_client)
+
     return CardContents(
         CARD_IDENTIFIER_FORMAT.format(seq_id=card.seq_id.raw_id),
         card.name,
@@ -84,7 +87,7 @@ def to_card_contents(card: Card, favro_client: FavroClient) -> CardContents:
         is_archived=card.is_archived,
         start_date=card.start_date,
         due_date=card.due_date,
-        custom_fields=to_custom_fields(card, favro_client),
+        custom_fields=custom_fields,
     )
 
 
@@ -157,6 +160,8 @@ def path_to_file_system_item(
     component = path_components[len(path)]
     return component.from_path_segment(path[-1] if path else None)
 
+FAST_GETATTR = True
+
 
 class FavroFuse(fuse.Fuse):
     """Favro FileSystem in Userspace."""
@@ -178,6 +183,7 @@ class FavroFuse(fuse.Fuse):
         super().__init__(**kwargs)
 
     def getattr(self, path: str) -> FavroStat | int:
+        logger.info('getattr: %s', path)
         file_system_item = path_to_file_system_item(path, self.path_components)
 
         st = FavroStat()
@@ -185,18 +191,26 @@ class FavroFuse(fuse.Fuse):
             st.st_mode = stat.S_IFDIR | 0o755
             st.st_nlink = 2
         elif isinstance(file_system_item, CardFileSystemItem):
-            card = self.favro_client.get_card(file_system_item.seq_id)
-
             st.st_mode = stat.S_IFREG | 0o666
             st.st_nlink = 1
-            st.st_size = len(self._format_card_file(card))
-            st.st_ctime = int(card.creation_date.timestamp())
+
+            card = self.favro_client.get_card_if_cached(file_system_item.seq_id)
+            if not FAST_GETATTR and card is None:
+                card = self.favro_client.get_card(file_system_item.seq_id)
+
+            if card is not None:
+                st.st_size = len(self._format_card_file(card))
+                st.st_ctime = int(card.creation_date.timestamp())
+            else:
+                st.st_size = len(path)
+
             st.st_mtime = st.st_ctime  # TODO
         else:
             return -errno.ENOENT
         return st
 
     def readdir(self, path: str, offset: int) -> Iterator[fuse.Direntry]:
+        logger.info('Reading directory: %s', path)
         file_system_item = path_to_file_system_item(path, self.path_components)
 
         yield fuse.Direntry('.')
@@ -219,13 +233,17 @@ class FavroFuse(fuse.Fuse):
                 yield fuse.Direntry(str(CardFileSystemItem(card.seq_id)))
                 del card
 
+        logger.info('Finished reading directory: %s', path)
+
     def open(self, path: str, flags: int) -> int | None:
+        logger.info('Opening: %s', path)
         file_system_item = path_to_file_system_item(path, self.path_components)
         if not isinstance(file_system_item, CardFileSystemItem):
             return -errno.ENOENT
         return None
 
     def read(self, path: str, size: int, offset: int) -> bytes | int:
+        logger.info('Reading: %s', path)
         # Check that this is a card file_system_item.
         file_system_item = path_to_file_system_item(path, self.path_components)
         if not isinstance(file_system_item, CardFileSystemItem):
@@ -246,6 +264,7 @@ class FavroFuse(fuse.Fuse):
         return buf
 
     def write(self, path: str, written_buffer: bytes, offset: int) -> int:
+        logger.info('Writing: %s', path)
         # Check that this is a card file_system_item.
         file_system_item = path_to_file_system_item(path, self.path_components)
         if not isinstance(file_system_item, CardFileSystemItem):
@@ -269,6 +288,7 @@ class FavroFuse(fuse.Fuse):
         return len(written_buffer)
 
     def truncate(self, path: str, new_size: int):
+        logger.info('Truncating: %s', path)
         # Check that this is a card file_system_item.
         file_system_item = path_to_file_system_item(path, self.path_components)
         if not isinstance(file_system_item, CardFileSystemItem):
@@ -318,6 +338,7 @@ Userspace hello example
 
 
 def start_favro_fuse(favro_client: FavroClient):
+    logger.info('Starting favro FUSE')
     # TODO:
     server = FavroFuse(
         favro_client=favro_client,

favro_sync/favro_markdown.py

@@ -1,11 +1,14 @@
 import dataclasses
 import datetime
 import re
+import logging
 
 import frontmatter
 import marko
 import marko.md_renderer
 
+logger = logging.getLogger(__name__)
+
 ################################################################################
 # FrontMatter keys
 
@@ -65,6 +68,7 @@ class CardFileFormatter:
 
     def format_card_contents(self, card: CardContents) -> str:
         """Formats card contents. Mostly the inverse of [`parse_card_contents`]."""
+        logger.info('Formatting card: %s', card.identifier)
         # Choose frontmatter data
         frontmatter_data = {}
         if self.obsidian_mode:
@@ -134,6 +138,8 @@ class CardFileFormatter:
         2. Parses header
         3. Finds content.
         """
+        logger.info('Parsing card contents (len %d)', len(contents))
+
         fm = frontmatter.loads(contents)
         del contents
 

ruff.toml

@@ -76,4 +76,5 @@ docstring-code-format = true
     "S101",    # Test Asserts
     "T201",    # Debug prints
     "PLR2004", # magic-value-comparison
+    'SLF001',  # Allow access to private members
 ]

setup.py

@@ -31,6 +31,7 @@ Features:
   - Tags
   - Assignees
   - Dependencies
+  - Custom fields
 - Change card features:
   - Title
   - Description
@@ -66,13 +67,22 @@ Limitations:
 
 Following features are work in progress:
 
-- [ ] Frontmatter: Update Tags
-- [ ] Frontmatter: Updated assigned members
-- [ ] Frontmatter: Arbitrary structured data? Read-only.
-- [ ] Frontmatter: Dependencies. As vault links in Obsidian mode.
-- [ ] Allow users to toggle Obsidian mode, instead of being default.
-- [ ] Get the correct last-modified date.
-- [ ] Improve cache behaviour. User and tags can have much longer cache times.
+- [ ] Frontmatter: Writable Tags
+- [ ] Frontmatter: Writable assigned members
+- [ ] Frontmatter: Writable Tasks.
+    1. Save updated TaskList along with card (using `PUT cards`)
+    2. Get the Card's TaskList.
+    3. Remove all TaskList's except for the latest (how to determine latest?)
+    4. That's three requests just to save a freaking list of tasks!
+- [ ] Frontmatter: Writable Dependencies.
+- [ ] Usability: Richer directory structure
+- [ ] Usability: Allow users to toggle Obsidian mode, instead of being default.
+- [ ] Precision: Get the correct last-modified date.
+- [ ] Performance: Improve cache behaviour. User and tags can have much longer cache times.
+- [ ] Performance: Parallelize requests.
+    * Paginated pages can be easily parallelize.
+- [X] Frontmatter: Arbitrary structured data (Custom Fields)? Read-only.
+- [X] Frontmatter: Readable Dependencies. As vault links in Obsidian mode.
 """.strip()
 
 PACKAGE_DESCRIPTION_SHORT = """
@@ -96,6 +106,7 @@ REQUIREMENTS_MAIN = [
     'requests-cache',
     'fuse-python',
     'secret_loader @ git+https://gitfub.space/Jmaa/secret_loader',
+    'requests_util @ git+https://gitfub.space/Jmaa/requests_util',
     'marko',
     'python-frontmatter',
 ]