From a476c6eac4dccfb6cd63ba808691bc0f28f1fbfd Mon Sep 17 00:00:00 2001 From: Jon Michael Aanes Date: Wed, 2 Oct 2024 17:17:14 +0200 Subject: [PATCH 01/10] [FUSE]: Collection directories WIP --- favro_sync/favro_client.py | 13 +++++------ favro_sync/favro_fuse.py | 45 +++++++++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/favro_sync/favro_client.py b/favro_sync/favro_client.py index 9da0946..37e227a 100644 --- a/favro_sync/favro_client.py +++ b/favro_sync/favro_client.py @@ -16,6 +16,7 @@ from .favro_data_model import ( CustomFieldId, CustomFieldInfo, OrganizationId, + CollectionId, SeqId, TagId, TagInfo, @@ -115,9 +116,10 @@ class FavroClient: self, *, seq_id: SeqId | None = None, + collection_id: CollectionId | None = None, todo_list=False, ) -> Iterator[Card]: - request = self._get_cards_request(seq_id=seq_id,todo_list=todo_list) + request = self._get_cards_request(seq_id=seq_id,todo_list=todo_list,collection_id=collection_id) yield from self._get_paginated(request, Card.from_json) def get_collections(self) -> Iterator[Collection]: @@ -134,18 +136,15 @@ class FavroClient: self, seq_id: SeqId | None = None, todo_list: bool = False, - request_id: None | str = None, - page: None | int = None, + collection_id: CollectionId | None = None, ) -> requests.Request: params = {'descriptionFormat': 'markdown'} if seq_id is not None: params['cardSequentialId'] = str(seq_id.raw_id) if todo_list is True: params['todoList'] = 'true' - if request_id: - params['requestId'] = request_id - if page: - params['page'] = page + if collection_id is not None: + params['collectionId'] = str(collection_id.raw_id) return requests.Request('GET', URL_GET_CARDS, params=params) diff --git a/favro_sync/favro_fuse.py b/favro_sync/favro_fuse.py index d92124f..c257a47 100644 --- a/favro_sync/favro_fuse.py +++ b/favro_sync/favro_fuse.py @@ -104,14 +104,19 @@ class RootFileSystemItem(FileSystemItem): def from_path_segment(segment: str) -> 'RootFileSystemItem': return RootFileSystemItem() + def __str__(self): + return '/' + @dataclasses.dataclass(frozen=True) -class OrganizationFileSystemItem(FileSystemItem): - name: str +class CollectionFileSystemItem(FileSystemItem): + collection_name: str @staticmethod - def from_path_segment(segment: str) -> 'OrganizationFileSystemItem': - return OrganizationFileSystemItem(segment) + def from_path_segment(segment: str) -> 'CollectionFileSystemItem': + return CollectionFileSystemItem(segment) + def __str__(self): + return self.collection_name @dataclasses.dataclass(frozen=True) class CardFileSystemItem(FileSystemItem): @@ -123,6 +128,9 @@ class CardFileSystemItem(FileSystemItem): return CardFileSystemItem(SeqId(int(m.group(1)))) return None + def __str__(self): + return CARD_FILENAME_FORMAT.format(seq_id=self.seq_id.raw_id) + def path_to_file_system_item(path_str: str, path_components: list[type[FileSystemItem]]) -> FileSystemItem | None: path = re.findall(r'[^/]+', path_str) @@ -142,15 +150,15 @@ class FavroFuse(fuse.Fuse): self.favro_client = favro_client self.formatter = formatter self.wiped_cards = set() - #self.path_components = [RootFileSystemItem, OrganizationFileSystemItem, CardFileSystemItem] - self.path_components = [RootFileSystemItem, CardFileSystemItem] + self.path_components = [RootFileSystemItem, CollectionFileSystemItem, CardFileSystemItem] super().__init__(**kwargs) def getattr(self, path: str) -> FavroStat | int: file_system_item = path_to_file_system_item(path, self.path_components) + print(file_system_item ) st = FavroStat() - if isinstance(file_system_item, RootFileSystemItem): + if isinstance(file_system_item, RootFileSystemItem | CollectionFileSystemItem): st.st_mode = stat.S_IFDIR | 0o755 st.st_nlink = 2 elif isinstance(file_system_item, CardFileSystemItem): @@ -166,12 +174,29 @@ class FavroFuse(fuse.Fuse): return st def readdir(self, path: str, offset: int) -> Iterator[fuse.Direntry]: - logger.warning('readdir(path=%s, offset=%s)', path, offset) + file_system_item = path_to_file_system_item(path, self.path_components) + yield fuse.Direntry('.') yield fuse.Direntry('..') - for card in self.favro_client.get_todo_list_cards(): - yield fuse.Direntry(CARD_FILENAME_FORMAT.format(seq_id=card.seq_id.raw_id)) + if isinstance(file_system_item, RootFileSystemItem): + for collection in self.favro_client.get_collections(): + print(collection) + yield fuse.Direntry(str(CollectionFileSystemItem(collection.name))) + del collection + elif isinstance(file_system_item, CollectionFileSystemItem): + + # TODO: move into own function + for collection in self.favro_client.get_collections(): + if collection.name == file_system_item.collection_name: + collection_id = collection.collection_id + del collection + + print('Collection', collection_id) + + for card in self.favro_client.get_cards(collection_id=collection_id): + yield fuse.Direntry(str(CardFileSystemItem(card.seq_id))) + del card def open(self, path: str, flags: int) -> int | None: file_system_item = path_to_file_system_item(path, self.path_components) From 32ad9f1476c2f788d136097c583d9dc39d70f65d Mon Sep 17 00:00:00 2001 From: Jon Michael Aanes Date: Tue, 8 Oct 2024 21:38:52 +0200 Subject: [PATCH 02/10] Fix collection names --- favro_sync/favro_fuse.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/favro_sync/favro_fuse.py b/favro_sync/favro_fuse.py index c257a47..e1e4e65 100644 --- a/favro_sync/favro_fuse.py +++ b/favro_sync/favro_fuse.py @@ -26,7 +26,7 @@ CARD_FILENAME_REGEX = r'^PAR\-(\d+)\.md$' ################################################################################ # Formatting -def to_custom_field_value(custom_field: CustomField, field_def: CustomFieldInfo) -> str: +def to_custom_field_value(custom_field: CustomField, field_def: CustomFieldInfo) -> str | None: value: CustomFieldItemId | list[CustomFieldItemId] = custom_field.value if field_def.type in {'Single select','Multiple select'}: items = [field_def.get_field_item(item_id) for item_id in value] @@ -34,17 +34,19 @@ def to_custom_field_value(custom_field: CustomField, field_def: CustomFieldInfo) return items[0].name if field_def.type in {'Color'}: return custom_field.color - assert False, 'Unknown type: ' + field_def.type + return None def to_custom_fields(card: Card, favro_client: FavroClient) -> dict[str,str]: custom_fields = {} for field_assignment in card.custom_fields: field_def = favro_client.get_custom_field(field_assignment.custom_field_id) - custom_fields[field_def.name] = to_custom_field_value(field_assignment, field_def) - del field_assignment + str_value = to_custom_field_value(field_assignment, field_def) + if str_value is not None: + custom_fields[field_def.name] = str_value + del field_assignment, str_value return custom_fields -def to_card_contents(card: Card, favro_client: FavroClient) -> str: +def to_card_contents(card: Card, favro_client: FavroClient) -> CardContents: tags = [favro_client.get_tag(tag_id).name for tag_id in card.tags] assignments = [ favro_client.get_user(assignment.user).name for assignment in card.assignments @@ -101,7 +103,7 @@ class FileSystemItem: class RootFileSystemItem(FileSystemItem): @staticmethod - def from_path_segment(segment: str) -> 'RootFileSystemItem': + def from_path_segment(_segment: str) -> 'RootFileSystemItem': return RootFileSystemItem() def __str__(self): @@ -111,6 +113,9 @@ class RootFileSystemItem(FileSystemItem): class CollectionFileSystemItem(FileSystemItem): collection_name: str + def __post_init__(self): + assert '/' not in self.collection_name + @staticmethod def from_path_segment(segment: str) -> 'CollectionFileSystemItem': return CollectionFileSystemItem(segment) @@ -155,7 +160,6 @@ class FavroFuse(fuse.Fuse): def getattr(self, path: str) -> FavroStat | int: file_system_item = path_to_file_system_item(path, self.path_components) - print(file_system_item ) st = FavroStat() if isinstance(file_system_item, RootFileSystemItem | CollectionFileSystemItem): @@ -181,19 +185,18 @@ class FavroFuse(fuse.Fuse): if isinstance(file_system_item, RootFileSystemItem): for collection in self.favro_client.get_collections(): - print(collection) - yield fuse.Direntry(str(CollectionFileSystemItem(collection.name))) + collection_name = collection.name.replace('/', '') + yield fuse.Direntry(str(CollectionFileSystemItem(collection_name))) del collection + elif isinstance(file_system_item, CollectionFileSystemItem): # TODO: move into own function for collection in self.favro_client.get_collections(): - if collection.name == file_system_item.collection_name: + if collection.name.replace('/', '') == file_system_item.collection_name: collection_id = collection.collection_id del collection - print('Collection', collection_id) - for card in self.favro_client.get_cards(collection_id=collection_id): yield fuse.Direntry(str(CardFileSystemItem(card.seq_id))) del card From 341630dcc9cd8091e29bcc420efaed754e2b781a Mon Sep 17 00:00:00 2001 From: Jon Michael Aanes Date: Thu, 10 Oct 2024 11:46:24 +0200 Subject: [PATCH 03/10] Format all --- test/test_integration_api_to_formatter.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 test/test_integration_api_to_formatter.py diff --git a/test/test_integration_api_to_formatter.py b/test/test_integration_api_to_formatter.py new file mode 100644 index 0000000..474985f --- /dev/null +++ b/test/test_integration_api_to_formatter.py @@ -0,0 +1,12 @@ +from favro_sync.favro_client import SeqId +from favro_sync.favro_fuse import to_card_contents +from favro_sync.favro_markdown import CardFileFormatter + +from .test_client import create_client, needs_secrets + +@needs_secrets +def test_format_all_cards(): + client = create_client() + for card in client.get_cards(todo_list=True): + contents = to_card_contents(card, client) + assert contents is not None From 809956059bcf213194941e2cc3b5dd18d04299f2 Mon Sep 17 00:00:00 2001 From: Jon Michael Aanes Date: Thu, 10 Oct 2024 13:51:29 +0200 Subject: [PATCH 04/10] API cache --- favro_sync/favro_client.py | 21 ++++++++++++++------- requirements.txt | 1 + 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/favro_sync/favro_client.py b/favro_sync/favro_client.py index 37e227a..1d24346 100644 --- a/favro_sync/favro_client.py +++ b/favro_sync/favro_client.py @@ -6,7 +6,9 @@ Implements methods for interacting with the [Favro API](https://favro.com/devel import dataclasses from collections.abc import Iterator from logging import getLogger +import datetime +import requests_util import requests from .favro_data_model import ( @@ -102,14 +104,19 @@ class FavroClient: ) self.read_only = read_only - self.cache = CardCache() + self.card_cache = CardCache() + + # Setup caching + requests_util.setup_limiter(self.session, URL_API_ROOT, datetime.timedelta(days=7)) + requests_util.setup_limiter(self.session, URL_GET_CARDS, datetime.timedelta(minutes=10)) + requests_util.setup_limiter(self.session, URL_GET_TASKS, datetime.timedelta(minutes=10)) def check_logged_in(self) -> None: next(self.get_todo_list_cards()) def get_todo_list_cards(self) -> Iterator[Card]: for card in self.get_cards(todo_list=True): - self.cache.add_card(card) + self.card_cache.add_card(card) yield card def get_cards( @@ -149,7 +156,7 @@ class FavroClient: return requests.Request('GET', URL_GET_CARDS, params=params) def get_card(self, seq_id: SeqId) -> Card: - if card := self.cache.get_card_by_seq_id(seq_id): + if card := self.card_cache.get_card_by_seq_id(seq_id): return card return next(self.get_cards(seq_id=seq_id)) @@ -172,9 +179,9 @@ class FavroClient: return CustomFieldInfo.from_json(response.json()) def _invalidate_cache(self, card_id: CardId) -> None: - card = self.cache.remove(card_id) + card = self.card_cache.remove(card_id) if card: - self.session.cache.delete( + self.session.card_cache.delete( requests=[self._get_cards_prepared_request(seq_id=card.seq_id)], ) @@ -183,7 +190,7 @@ class FavroClient: card_id: CardId, card_contents: CardContents, ) -> Card | None: - if card := self.cache.remove(card_id): + if card := self.card_cache.remove(card_id): card = dataclasses.replace( card, detailed_description=card_contents.description, @@ -192,7 +199,7 @@ class FavroClient: assignments=[], # TODO? dependencies=[], # TODO ) - self.cache.add_card(card) + self.card_cache.add_card(card) return card def update_card_contents( diff --git a/requirements.txt b/requirements.txt index bf57780..31866af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,6 @@ requests requests-cache fuse-python secret_loader @ git+https://gitfub.space/Jmaa/secret_loader +requests_util @ git+https://gitfub.space/Jmaa/requests_util marko python-frontmatter From e594dff1496be30010d6a16139c078d5552f2325 Mon Sep 17 00:00:00 2001 From: Jon Michael Aanes Date: Thu, 10 Oct 2024 14:00:47 +0200 Subject: [PATCH 05/10] Improved caching --- favro_sync/favro_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/favro_sync/favro_client.py b/favro_sync/favro_client.py index 1d24346..80ad53f 100644 --- a/favro_sync/favro_client.py +++ b/favro_sync/favro_client.py @@ -115,9 +115,7 @@ class FavroClient: next(self.get_todo_list_cards()) def get_todo_list_cards(self) -> Iterator[Card]: - for card in self.get_cards(todo_list=True): - self.card_cache.add_card(card) - yield card + yield from self.get_cards(todo_list=True) def get_cards( self, @@ -127,7 +125,9 @@ class FavroClient: todo_list=False, ) -> Iterator[Card]: request = self._get_cards_request(seq_id=seq_id,todo_list=todo_list,collection_id=collection_id) - yield from self._get_paginated(request, Card.from_json) + for card in self._get_paginated(request, Card.from_json): + self.card_cache.add_card(card) + yield card def get_collections(self) -> Iterator[Collection]: request = requests.Request('GET', URL_GET_COLLECTIONS) From cbb0cba076f69e1774bd0e3d1e17dcf1d40bf367 Mon Sep 17 00:00:00 2001 From: Jon Michael Aanes Date: Thu, 10 Oct 2024 14:02:02 +0200 Subject: [PATCH 06/10] Ruff --- favro_sync/favro_client.py | 31 ++++++++++++------ favro_sync/favro_data_model.py | 40 +++++++++++++---------- favro_sync/favro_fuse.py | 34 ++++++++++++++----- favro_sync/favro_markdown.py | 4 +-- test/test_client.py | 4 ++- test/test_integration_api_to_formatter.py | 3 +- 6 files changed, 76 insertions(+), 40 deletions(-) diff --git a/favro_sync/favro_client.py b/favro_sync/favro_client.py index 80ad53f..8b81962 100644 --- a/favro_sync/favro_client.py +++ b/favro_sync/favro_client.py @@ -4,21 +4,21 @@ Implements methods for interacting with the [Favro API](https://favro.com/devel """ import dataclasses +import datetime from collections.abc import Iterator from logging import getLogger -import datetime -import requests_util import requests +import requests_util from .favro_data_model import ( Card, CardId, Collection, + CollectionId, CustomFieldId, CustomFieldInfo, OrganizationId, - CollectionId, SeqId, TagId, TagInfo, @@ -107,9 +107,15 @@ class FavroClient: self.card_cache = CardCache() # Setup caching - requests_util.setup_limiter(self.session, URL_API_ROOT, datetime.timedelta(days=7)) - requests_util.setup_limiter(self.session, URL_GET_CARDS, datetime.timedelta(minutes=10)) - requests_util.setup_limiter(self.session, URL_GET_TASKS, datetime.timedelta(minutes=10)) + requests_util.setup_limiter( + self.session, URL_API_ROOT, datetime.timedelta(days=7), + ) + requests_util.setup_limiter( + self.session, URL_GET_CARDS, datetime.timedelta(minutes=10), + ) + requests_util.setup_limiter( + self.session, URL_GET_TASKS, datetime.timedelta(minutes=10), + ) def check_logged_in(self) -> None: next(self.get_todo_list_cards()) @@ -124,7 +130,9 @@ class FavroClient: collection_id: CollectionId | None = None, todo_list=False, ) -> Iterator[Card]: - request = self._get_cards_request(seq_id=seq_id,todo_list=todo_list,collection_id=collection_id) + request = self._get_cards_request( + seq_id=seq_id, todo_list=todo_list, collection_id=collection_id, + ) for card in self._get_paginated(request, Card.from_json): self.card_cache.add_card(card) yield card @@ -134,9 +142,10 @@ class FavroClient: yield from self._get_paginated(request, Collection.from_json) def _get_cards_prepared_request( - self, **kwargs, + self, + **kwargs, ) -> requests.PreparedRequest: - request = self._get_cards_request(**kwargs) + request = self._get_cards_request(**kwargs) return self.session.prepare_request(request) def _get_cards_request( @@ -234,7 +243,9 @@ class FavroClient: self._invalidate_cache(card_id) return self.update_card_contents_locally(card_id, card_contents) - def _get_paginated(self, base_request: requests.Request, entity_from_json) -> Iterator: + def _get_paginated( + self, base_request: requests.Request, entity_from_json, + ) -> Iterator: page = 0 request_id = None num_pages = 1 diff --git a/favro_sync/favro_data_model.py b/favro_sync/favro_data_model.py index 4346a41..f06505b 100644 --- a/favro_sync/favro_data_model.py +++ b/favro_sync/favro_data_model.py @@ -136,8 +136,8 @@ class CustomFieldItem: @staticmethod def from_json(json: dict[str, Any]) -> 'CustomFieldItem': return CustomFieldItem( - custom_field_item_id=CustomFieldItemId(json['customFieldItemId']), - name=json['name'], + custom_field_item_id=CustomFieldItemId(json['customFieldItemId']), + name=json['name'], ) @@ -163,7 +163,9 @@ class CustomFieldInfo: enabled: bool custom_field_items: list[CustomFieldItem] - def get_field_item(self, field_item_id: CustomFieldItemId) -> CustomFieldItem | None: + def get_field_item( + self, field_item_id: CustomFieldItemId, + ) -> CustomFieldItem | None: for item in self.custom_field_items: if item.custom_field_item_id == field_item_id: return item @@ -174,11 +176,13 @@ class CustomFieldInfo: return CustomFieldInfo( organization_id=OrganizationId(json['organizationId']), custom_field_id=CustomFieldId(json['customFieldId']), - widget_common_id=map_opt(WidgetCommonId,json.get('widgetCommonId')), + widget_common_id=map_opt(WidgetCommonId, json.get('widgetCommonId')), type=json['type'], name=json['name'], enabled=json['enabled'], - custom_field_items=[CustomFieldItem.from_json(f) for f in json.get('customFieldItems', [])], + custom_field_items=[ + CustomFieldItem.from_json(f) for f in json.get('customFieldItems', []) + ], ) @@ -198,9 +202,9 @@ class CustomField: else: typed_value = [CustomFieldItemId(v) for v in value] return CustomField( - custom_field_id = CustomFieldId(json['customFieldId']), - value = typed_value, - color = json.get('color'), + custom_field_id=CustomFieldId(json['customFieldId']), + value=typed_value, + color=json.get('color'), ) @@ -243,12 +247,13 @@ class Task: position=json['position'], ) + @dataclasses.dataclass(frozen=True) class Collection: collection_id: CollectionId organization_id: OrganizationId name: str - shared_to_users: list[dict[str,str]] # TODO + shared_to_users: list[dict[str, str]] # TODO public_sharing: str background: str is_archived: bool @@ -257,16 +262,17 @@ class Collection: @staticmethod def from_json(json: dict[str, Any]) -> 'Collection': return Collection( - collection_id=CollectionId(json['collectionId']), - organization_id=OrganizationId(json['collectionId']), - name=json['name'], - shared_to_users=json['sharedToUsers'], - public_sharing=json['publicSharing'], - background=json['background'], - is_archived=json['archived'], - widget_common_id=map_opt(WidgetCommonId,json.get('widgetCommonId')), + collection_id=CollectionId(json['collectionId']), + organization_id=OrganizationId(json['collectionId']), + name=json['name'], + shared_to_users=json['sharedToUsers'], + public_sharing=json['publicSharing'], + background=json['background'], + is_archived=json['archived'], + widget_common_id=map_opt(WidgetCommonId, json.get('widgetCommonId')), ) + @dataclasses.dataclass(frozen=True) class Card: card_id: CardId diff --git a/favro_sync/favro_fuse.py b/favro_sync/favro_fuse.py index e1e4e65..2ac3029 100644 --- a/favro_sync/favro_fuse.py +++ b/favro_sync/favro_fuse.py @@ -8,7 +8,13 @@ from logging import getLogger import fuse from .favro_client import FavroClient -from .favro_data_model import Card, SeqId, CustomFieldInfo, CustomFieldItemId, CustomField +from .favro_data_model import ( + Card, + CustomField, + CustomFieldInfo, + CustomFieldItemId, + SeqId, +) from .favro_markdown import CardContents, CardFileFormatter ################################################################################ @@ -26,9 +32,12 @@ CARD_FILENAME_REGEX = r'^PAR\-(\d+)\.md$' ################################################################################ # Formatting -def to_custom_field_value(custom_field: CustomField, field_def: CustomFieldInfo) -> str | None: + +def to_custom_field_value( + custom_field: CustomField, field_def: CustomFieldInfo, +) -> str | None: value: CustomFieldItemId | list[CustomFieldItemId] = custom_field.value - if field_def.type in {'Single select','Multiple select'}: + if field_def.type in {'Single select', 'Multiple select'}: items = [field_def.get_field_item(item_id) for item_id in value] items = [i for i in items if i] return items[0].name @@ -36,7 +45,8 @@ def to_custom_field_value(custom_field: CustomField, field_def: CustomFieldInfo) return custom_field.color return None -def to_custom_fields(card: Card, favro_client: FavroClient) -> dict[str,str]: + +def to_custom_fields(card: Card, favro_client: FavroClient) -> dict[str, str]: custom_fields = {} for field_assignment in card.custom_fields: field_def = favro_client.get_custom_field(field_assignment.custom_field_id) @@ -46,6 +56,7 @@ def to_custom_fields(card: Card, favro_client: FavroClient) -> dict[str,str]: del field_assignment, str_value return custom_fields + def to_card_contents(card: Card, favro_client: FavroClient) -> CardContents: tags = [favro_client.get_tag(tag_id).name for tag_id in card.tags] assignments = [ @@ -80,6 +91,7 @@ def to_card_contents(card: Card, favro_client: FavroClient) -> CardContents: ################################################################################ # FUSE + class FavroStat(fuse.Stat): def __init__(self): self.st_mode = 0 @@ -101,7 +113,6 @@ class FileSystemItem: @dataclasses.dataclass(frozen=True) class RootFileSystemItem(FileSystemItem): - @staticmethod def from_path_segment(_segment: str) -> 'RootFileSystemItem': return RootFileSystemItem() @@ -109,6 +120,7 @@ class RootFileSystemItem(FileSystemItem): def __str__(self): return '/' + @dataclasses.dataclass(frozen=True) class CollectionFileSystemItem(FileSystemItem): collection_name: str @@ -123,6 +135,7 @@ class CollectionFileSystemItem(FileSystemItem): def __str__(self): return self.collection_name + @dataclasses.dataclass(frozen=True) class CardFileSystemItem(FileSystemItem): seq_id: SeqId @@ -137,7 +150,9 @@ class CardFileSystemItem(FileSystemItem): return CARD_FILENAME_FORMAT.format(seq_id=self.seq_id.raw_id) -def path_to_file_system_item(path_str: str, path_components: list[type[FileSystemItem]]) -> FileSystemItem | None: +def path_to_file_system_item( + path_str: str, path_components: list[type[FileSystemItem]], +) -> FileSystemItem | None: path = re.findall(r'[^/]+', path_str) component = path_components[len(path)] return component.from_path_segment(path[-1] if path else None) @@ -155,7 +170,11 @@ class FavroFuse(fuse.Fuse): self.favro_client = favro_client self.formatter = formatter self.wiped_cards = set() - self.path_components = [RootFileSystemItem, CollectionFileSystemItem, CardFileSystemItem] + self.path_components = [ + RootFileSystemItem, + CollectionFileSystemItem, + CardFileSystemItem, + ] super().__init__(**kwargs) def getattr(self, path: str) -> FavroStat | int: @@ -190,7 +209,6 @@ class FavroFuse(fuse.Fuse): del collection elif isinstance(file_system_item, CollectionFileSystemItem): - # TODO: move into own function for collection in self.favro_client.get_collections(): if collection.name.replace('/', '') == file_system_item.collection_name: diff --git a/favro_sync/favro_markdown.py b/favro_sync/favro_markdown.py index 4b464ea..062597a 100644 --- a/favro_sync/favro_markdown.py +++ b/favro_sync/favro_markdown.py @@ -35,7 +35,7 @@ class CardContents: is_archived: bool start_date: datetime.date | None due_date: datetime.date | None - custom_fields: dict[str,str] + custom_fields: dict[str, str] def format_obsidian_link(text: str) -> str: @@ -173,5 +173,5 @@ class CardFileFormatter: is_archived=is_archived, start_date=start_date, due_date=due_date, - custom_fields={}, # TODO + custom_fields={}, # TODO ) diff --git a/test/test_client.py b/test/test_client.py index 6cacdd0..b06f7cb 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -1,6 +1,6 @@ import pytest - import requests_cache + from favro_sync import secrets from favro_sync.favro_client import FavroClient, OrganizationId, SeqId @@ -50,6 +50,7 @@ def test_get_cards(): for card in client.get_cards(todo_list=True): assert_valid_card(card) + @needs_secrets def test_get_collections(): client = create_client() @@ -62,6 +63,7 @@ def test_get_collections(): assert collection.is_archived is not None assert collection.widget_common_id is None + def create_client(): session = requests_cache.CachedSession('output/test-http-cache.sqlite') return FavroClient( diff --git a/test/test_integration_api_to_formatter.py b/test/test_integration_api_to_formatter.py index 474985f..f8543e6 100644 --- a/test/test_integration_api_to_formatter.py +++ b/test/test_integration_api_to_formatter.py @@ -1,9 +1,8 @@ -from favro_sync.favro_client import SeqId from favro_sync.favro_fuse import to_card_contents -from favro_sync.favro_markdown import CardFileFormatter from .test_client import create_client, needs_secrets + @needs_secrets def test_format_all_cards(): client = create_client() From 7d5e1ec088daa69341e8cac2fce772378b66f0da Mon Sep 17 00:00:00 2001 From: takunomi-build-bot Date: Thu, 19 Dec 2024 14:00:09 +0100 Subject: [PATCH 07/10] =?UTF-8?q?=F0=9F=A4=96=20Repository=20layout=20upda?= =?UTF-8?q?ted=20to=20latest=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit was automatically generated by a script: https://gitfub.space/Jmaa/repo-manager --- .gitea/workflows/python-package.yml | 2 +- .gitea/workflows/python-test.yml | 26 ++++++++++++++++++-- .gitea/workflows/python-version-check.yml | 28 +++++++++++++++++++++ README.md | 30 ++++++++++++++++------- ruff.toml | 1 + setup.py | 25 +++++++++++++------ 6 files changed, 93 insertions(+), 19 deletions(-) create mode 100644 .gitea/workflows/python-version-check.yml diff --git a/.gitea/workflows/python-package.yml b/.gitea/workflows/python-package.yml index c29c732..09275ea 100644 --- a/.gitea/workflows/python-package.yml +++ b/.gitea/workflows/python-package.yml @@ -10,7 +10,7 @@ jobs: uses: jmaa/workflows/.gitea/workflows/python-package.yaml@v6.21 with: REGISTRY_DOMAIN: gitfub.space - REGISTRY_ORGANIZATION: usagi-keiretsu + REGISTRY_ORGANIZATION: jmaa secrets: PIPY_REPO_USER: ${{ secrets.PIPY_REPO_USER }} PIPY_REPO_PASS: ${{ secrets.PIPY_REPO_PASS }} diff --git a/.gitea/workflows/python-test.yml b/.gitea/workflows/python-test.yml index d85fe9a..18b55db 100644 --- a/.gitea/workflows/python-test.yml +++ b/.gitea/workflows/python-test.yml @@ -1,4 +1,4 @@ -name: Test Python +name: Run Python tests (through Pytest) on: push: @@ -6,4 +6,26 @@ on: jobs: Test: - uses: jmaa/workflows/.gitea/workflows/python-test.yaml@v6.21 + runs-on: ubuntu-latest + container: + image: node:21-bookworm + steps: + - name: Setting up Python ${{ env.PYTHON_VERSION }} for ${{runner.arch}} ${{runner.os}} + run: | + apt-get update + apt-get install -y python3 python3-pip + - name: Check out repository code + if: success() + uses: actions/checkout@v3 + - name: Installing Python Dependencies + if: success() + run: python3 -m pip install --upgrade pip setuptools wheel build twine pytest pytest-cov --break-system-packages + - name: Installing Python Test Dependencies + if: success() && hashFiles('requirements_test.txt') != '' + run: python3 -m pip install --upgrade -r requirements_test.txt --break-system-packages + - name: Installing package + if: success() + run: python3 -m pip install .[test] --break-system-packages + - name: Test Python Code + if: success() + run: python3 -m pytest test --cov=favro_sync --cov-report html:htmlcov --cov-fail-under=0 diff --git a/.gitea/workflows/python-version-check.yml b/.gitea/workflows/python-version-check.yml new file mode 100644 index 0000000..998a20c --- /dev/null +++ b/.gitea/workflows/python-version-check.yml @@ -0,0 +1,28 @@ +name: Verify Python project can be installed, loaded and have version checked + +on: + push: + paths-ignore: ["README.md", ".gitignore", "LICENSE", "ruff.toml"] + +jobs: + Test: + runs-on: ubuntu-latest + container: + image: node:21-bookworm + steps: + - name: Setting up Python ${{ env.PYTHON_VERSION }} for ${{runner.arch}} ${{runner.os}} + run: | + apt-get update + apt-get install -y python3 python3-pip + - name: Check out repository code + if: success() + uses: actions/checkout@v3 + - name: Installing Python Dependencies + if: success() + run: python3 -m pip install --upgrade pip setuptools wheel build twine --break-system-packages + - name: Installing package + if: success() + run: python3 -m pip install . --break-system-packages + - name: Check version field + if: success() + run: python3 -c "import favro_sync; assert favro_sync.__version__ is not None" diff --git a/README.md b/README.md index c1df6ad..d0274ca 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,9 @@ -# Favro Sync. +# Favro Sync + +![Test program/library](https://gitfub.space/Jmaa/favro-sync/actions/workflows/python-test.yml/badge.svg) Filesystem in User Space for Favro. @@ -24,6 +26,7 @@ Features: - Tags - Assignees - Dependencies + - Custom fields - Change card features: - Title - Description @@ -59,14 +62,22 @@ Limitations: Following features are work in progress: -- [ ] Frontmatter: Update Tags -- [ ] Frontmatter: Updated assigned members -- [ ] Frontmatter: Arbitrary structured data? Read-only. -- [ ] Frontmatter: Dependencies. As vault links in Obsidian mode. -- [ ] Allow users to toggle Obsidian mode, instead of being default. -- [ ] Get the correct last-modified date. -- [ ] Improve cache behaviour. User and tags can have much longer cache times. - +- [ ] Frontmatter: Writable Tags +- [ ] Frontmatter: Writable assigned members +- [ ] Frontmatter: Writable Tasks. + 1. Save updated TaskList along with card (using `PUT cards`) + 2. Get the Card's TaskList. + 3. Remove all TaskList's except for the latest (how to determine latest?) + 4. That's three requests just to save a freaking list of tasks! +- [ ] Frontmatter: Writable Dependencies. +- [ ] Usability: Richer directory structure +- [ ] Usability: Allow users to toggle Obsidian mode, instead of being default. +- [ ] Precision: Get the correct last-modified date. +- [ ] Performance: Improve cache behaviour. User and tags can have much longer cache times. +- [ ] Performance: Parallelize requests. + * Paginated pages can be easily parallelize. +- [X] Frontmatter: Arbitrary structured data (Custom Fields)? Read-only. +- [X] Frontmatter: Readable Dependencies. As vault links in Obsidian mode. ## Dependencies @@ -81,6 +92,7 @@ Full list of requirements: - [requests-cache](https://pypi.org/project/requests-cache/) - [fuse-python](https://pypi.org/project/fuse-python/) - [secret_loader](https://gitfub.space/Jmaa/secret_loader) +- [requests_util](https://gitfub.space/Jmaa/requests_util) - [marko](https://pypi.org/project/marko/) - [python-frontmatter](https://pypi.org/project/python-frontmatter/) diff --git a/ruff.toml b/ruff.toml index 317a9a2..c397e12 100644 --- a/ruff.toml +++ b/ruff.toml @@ -76,4 +76,5 @@ docstring-code-format = true "S101", # Test Asserts "T201", # Debug prints "PLR2004", # magic-value-comparison + 'SLF001', # Allow access to private members ] diff --git a/setup.py b/setup.py index 88df9aa..09c53cb 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ Features: - Tags - Assignees - Dependencies + - Custom fields - Change card features: - Title - Description @@ -66,13 +67,22 @@ Limitations: Following features are work in progress: -- [ ] Frontmatter: Update Tags -- [ ] Frontmatter: Updated assigned members -- [ ] Frontmatter: Arbitrary structured data? Read-only. -- [ ] Frontmatter: Dependencies. As vault links in Obsidian mode. -- [ ] Allow users to toggle Obsidian mode, instead of being default. -- [ ] Get the correct last-modified date. -- [ ] Improve cache behaviour. User and tags can have much longer cache times. +- [ ] Frontmatter: Writable Tags +- [ ] Frontmatter: Writable assigned members +- [ ] Frontmatter: Writable Tasks. + 1. Save updated TaskList along with card (using `PUT cards`) + 2. Get the Card's TaskList. + 3. Remove all TaskList's except for the latest (how to determine latest?) + 4. That's three requests just to save a freaking list of tasks! +- [ ] Frontmatter: Writable Dependencies. +- [ ] Usability: Richer directory structure +- [ ] Usability: Allow users to toggle Obsidian mode, instead of being default. +- [ ] Precision: Get the correct last-modified date. +- [ ] Performance: Improve cache behaviour. User and tags can have much longer cache times. +- [ ] Performance: Parallelize requests. + * Paginated pages can be easily parallelize. +- [X] Frontmatter: Arbitrary structured data (Custom Fields)? Read-only. +- [X] Frontmatter: Readable Dependencies. As vault links in Obsidian mode. """.strip() PACKAGE_DESCRIPTION_SHORT = """ @@ -96,6 +106,7 @@ REQUIREMENTS_MAIN = [ 'requests-cache', 'fuse-python', 'secret_loader @ git+https://gitfub.space/Jmaa/secret_loader', + 'requests_util @ git+https://gitfub.space/Jmaa/requests_util', 'marko', 'python-frontmatter', ] From 2c8a65f959e870a5377b45d6c903ad528cf9c0f5 Mon Sep 17 00:00:00 2001 From: Jon Michael Aanes Date: Wed, 8 Jan 2025 14:05:17 +0100 Subject: [PATCH 08/10] Cache users aggressively --- favro_sync/__init__.py | 2 ++ favro_sync/__main__.py | 1 + favro_sync/favro_client.py | 55 ++++++++++++++++++++++++++++-------- favro_sync/favro_fuse.py | 31 ++++++++++++++++---- favro_sync/favro_markdown.py | 6 ++++ 5 files changed, 79 insertions(+), 16 deletions(-) diff --git a/favro_sync/__init__.py b/favro_sync/__init__.py index 082e236..758155c 100644 --- a/favro_sync/__init__.py +++ b/favro_sync/__init__.py @@ -43,6 +43,8 @@ Limitations: [`python-fuse`](https://github.com/libfuse/python-fuse) implements a whole bunch automatically.) +Use `umount` to unmount the cards again. + ## Architecture - `FavroFuse` diff --git a/favro_sync/__main__.py b/favro_sync/__main__.py index e500928..b0bef18 100644 --- a/favro_sync/__main__.py +++ b/favro_sync/__main__.py @@ -10,6 +10,7 @@ from .favro_fuse import start_favro_fuse def main(): logging.basicConfig() + logging.getLogger().setLevel('INFO') read_only = False diff --git a/favro_sync/favro_client.py b/favro_sync/favro_client.py index 8b81962..ecd9ff7 100644 --- a/favro_sync/favro_client.py +++ b/favro_sync/favro_client.py @@ -4,6 +4,8 @@ Implements methods for interacting with the [Favro API](https://favro.com/devel """ import dataclasses +from typing import Any +import requests_cache import datetime from collections.abc import Iterator from logging import getLogger @@ -33,6 +35,7 @@ logger = getLogger(__name__) URL_API_ROOT = 'https://favro.com/api/v1' URL_GET_CARDS = URL_API_ROOT + '/cards' URL_UPDATE_CARD = URL_API_ROOT + '/cards/{card_id}' +URL_GET_USERS = URL_API_ROOT + '/users/' URL_GET_USER = URL_API_ROOT + '/users/{user_id}' URL_GET_TAG = URL_API_ROOT + '/tags/{tag_id}' URL_GET_CUSTOM_FIELD = URL_API_ROOT + '/customfields/{custom_field_id}' @@ -105,6 +108,7 @@ class FavroClient: self.read_only = read_only self.card_cache = CardCache() + self.user_cache = None # Setup caching requests_util.setup_limiter( @@ -116,6 +120,9 @@ class FavroClient: requests_util.setup_limiter( self.session, URL_GET_TASKS, datetime.timedelta(minutes=10), ) + requests_util.setup_limiter( + self.session, URL_GET_CUSTOM_FIELD, datetime.timedelta(days=30), + ) def check_logged_in(self) -> None: next(self.get_todo_list_cards()) @@ -130,6 +137,7 @@ class FavroClient: collection_id: CollectionId | None = None, todo_list=False, ) -> Iterator[Card]: + logger.info('Getting cards: seq_id=%s, collection_id=%s, todo_list=%s', seq_id, collection_id, todo_list) request = self._get_cards_request( seq_id=seq_id, todo_list=todo_list, collection_id=collection_id, ) @@ -165,27 +173,44 @@ class FavroClient: return requests.Request('GET', URL_GET_CARDS, params=params) def get_card(self, seq_id: SeqId) -> Card: - if card := self.card_cache.get_card_by_seq_id(seq_id): + if card := self.get_card_if_cached(seq_id): return card return next(self.get_cards(seq_id=seq_id)) + def get_card_if_cached(self, seq_id: SeqId) -> Card | None: + if card := self.card_cache.get_card_by_seq_id(seq_id): + return card + return None + def get_card_by_card_id(self, card_id: CardId) -> Card: - response = self.session.get(URL_UPDATE_CARD.format(card_id=card_id.raw_id)) - return Card.from_json(response.json()) + json_data = self._get_json(URL_UPDATE_CARD.format(card_id=card_id.raw_id)) + return Card.from_json(json_data) def get_user(self, user_id: UserId) -> UserInfo: - response = self.session.get(URL_GET_USER.format(user_id=user_id.raw_id)) - return UserInfo.from_json(response.json()) + self._load_users() + return self.user_cache[user_id] + + def _load_users(self) -> None: + if self.user_cache is not None: + return + + request = requests.Request('GET', URL_GET_USERS) + + user_cache = {} + for user_info in self._get_paginated(request, UserInfo.from_json): + user_cache[user_info.user_id] = user_info + self.user_cache = user_cache + logger.info('Loaded %s users', len(user_cache)) def get_tag(self, tag_id: TagId) -> TagInfo: - response = self.session.get(URL_GET_TAG.format(tag_id=tag_id.raw_id)) - return TagInfo.from_json(response.json()) + json_data = self._get_json(URL_GET_TAG.format(tag_id=tag_id.raw_id)) + return TagInfo.from_json(json_data) def get_custom_field(self, custom_field_id: CustomFieldId) -> CustomFieldInfo: - response = self.session.get( + json_data = self._get_json( URL_GET_CUSTOM_FIELD.format(custom_field_id=custom_field_id.raw_id), ) - return CustomFieldInfo.from_json(response.json()) + return CustomFieldInfo.from_json(json_data) def _invalidate_cache(self, card_id: CardId) -> None: card = self.card_cache.remove(card_id) @@ -234,7 +259,7 @@ class FavroClient: } url = URL_UPDATE_CARD.format(card_id=card_id.raw_id) - logger.warning('Sending request: %s', url) + logger.info('Sending PUT: %s', url) response = self.session.put( url, json=json_body, @@ -257,9 +282,11 @@ class FavroClient: request = self.session.prepare_request(base_request) # Run query - logger.warning('Sending request: %s', request.url) + logger.debug('Sending GET: %s', request.url) response = self.session.send(request) response.raise_for_status() + if not response.from_cache: + logger.warning('Got new result: %s', request.url) json = response.json() for entity_json in json['entities']: @@ -271,3 +298,9 @@ class FavroClient: page = json['page'] + 1 request_id = json['requestId'] num_pages = json['pages'] + + def _get_json(self, url: str) -> dict[str, Any]: + logger.debug('Sending GET: %s', url) + response = self.session.get(url) + response.raise_for_status() + return response.json() diff --git a/favro_sync/favro_fuse.py b/favro_sync/favro_fuse.py index 2ac3029..1b4ef66 100644 --- a/favro_sync/favro_fuse.py +++ b/favro_sync/favro_fuse.py @@ -58,6 +58,7 @@ def to_custom_fields(card: Card, favro_client: FavroClient) -> dict[str, str]: def to_card_contents(card: Card, favro_client: FavroClient) -> CardContents: + logger.info('Getting card contents for card: PAR-%s', card.seq_id) tags = [favro_client.get_tag(tag_id).name for tag_id in card.tags] assignments = [ favro_client.get_user(assignment.user).name for assignment in card.assignments @@ -69,6 +70,8 @@ def to_card_contents(card: Card, favro_client: FavroClient) -> CardContents: for dep in card.dependencies if dep.is_before ] + custom_fields = to_custom_fields(card, favro_client) + return CardContents( CARD_IDENTIFIER_FORMAT.format(seq_id=card.seq_id.raw_id), card.name, @@ -84,7 +87,7 @@ def to_card_contents(card: Card, favro_client: FavroClient) -> CardContents: is_archived=card.is_archived, start_date=card.start_date, due_date=card.due_date, - custom_fields=to_custom_fields(card, favro_client), + custom_fields=custom_fields, ) @@ -157,6 +160,8 @@ def path_to_file_system_item( component = path_components[len(path)] return component.from_path_segment(path[-1] if path else None) +FAST_GETATTR = True + class FavroFuse(fuse.Fuse): """Favro FileSystem in Userspace.""" @@ -178,6 +183,7 @@ class FavroFuse(fuse.Fuse): super().__init__(**kwargs) def getattr(self, path: str) -> FavroStat | int: + logger.info('getattr: %s', path) file_system_item = path_to_file_system_item(path, self.path_components) st = FavroStat() @@ -185,18 +191,26 @@ class FavroFuse(fuse.Fuse): st.st_mode = stat.S_IFDIR | 0o755 st.st_nlink = 2 elif isinstance(file_system_item, CardFileSystemItem): - card = self.favro_client.get_card(file_system_item.seq_id) - st.st_mode = stat.S_IFREG | 0o666 st.st_nlink = 1 - st.st_size = len(self._format_card_file(card)) - st.st_ctime = int(card.creation_date.timestamp()) + + card = self.favro_client.get_card_if_cached(file_system_item.seq_id) + if not FAST_GETATTR and card is None: + card = self.favro_client.get_card(file_system_item.seq_id) + + if card is not None: + st.st_size = len(self._format_card_file(card)) + st.st_ctime = int(card.creation_date.timestamp()) + else: + st.st_size = len(path) + st.st_mtime = st.st_ctime # TODO else: return -errno.ENOENT return st def readdir(self, path: str, offset: int) -> Iterator[fuse.Direntry]: + logger.info('Reading directory: %s', path) file_system_item = path_to_file_system_item(path, self.path_components) yield fuse.Direntry('.') @@ -219,13 +233,17 @@ class FavroFuse(fuse.Fuse): yield fuse.Direntry(str(CardFileSystemItem(card.seq_id))) del card + logger.info('Finished reading directory: %s', path) + def open(self, path: str, flags: int) -> int | None: + logger.info('Opening: %s', path) file_system_item = path_to_file_system_item(path, self.path_components) if not isinstance(file_system_item, CardFileSystemItem): return -errno.ENOENT return None def read(self, path: str, size: int, offset: int) -> bytes | int: + logger.info('Reading: %s', path) # Check that this is a card file_system_item. file_system_item = path_to_file_system_item(path, self.path_components) if not isinstance(file_system_item, CardFileSystemItem): @@ -246,6 +264,7 @@ class FavroFuse(fuse.Fuse): return buf def write(self, path: str, written_buffer: bytes, offset: int) -> int: + logger.info('Writing: %s', path) # Check that this is a card file_system_item. file_system_item = path_to_file_system_item(path, self.path_components) if not isinstance(file_system_item, CardFileSystemItem): @@ -269,6 +288,7 @@ class FavroFuse(fuse.Fuse): return len(written_buffer) def truncate(self, path: str, new_size: int): + logger.info('Truncating: %s', path) # Check that this is a card file_system_item. file_system_item = path_to_file_system_item(path, self.path_components) if not isinstance(file_system_item, CardFileSystemItem): @@ -318,6 +338,7 @@ Userspace hello example def start_favro_fuse(favro_client: FavroClient): + logger.info('Starting favro FUSE') # TODO: server = FavroFuse( favro_client=favro_client, diff --git a/favro_sync/favro_markdown.py b/favro_sync/favro_markdown.py index 062597a..d80f1fa 100644 --- a/favro_sync/favro_markdown.py +++ b/favro_sync/favro_markdown.py @@ -1,11 +1,14 @@ import dataclasses import datetime import re +import logging import frontmatter import marko import marko.md_renderer +logger = logging.getLogger(__name__) + ################################################################################ # FrontMatter keys @@ -65,6 +68,7 @@ class CardFileFormatter: def format_card_contents(self, card: CardContents) -> str: """Formats card contents. Mostly the inverse of [`parse_card_contents`].""" + logger.info('Formatting card: %s', card.identifier) # Choose frontmatter data frontmatter_data = {} if self.obsidian_mode: @@ -134,6 +138,8 @@ class CardFileFormatter: 2. Parses header 3. Finds content. """ + logger.info('Parsing card contents (len %d)', len(contents)) + fm = frontmatter.loads(contents) del contents From 7a42938523b937238a2a75f2d13f2d26cebeccde Mon Sep 17 00:00:00 2001 From: Jon Michael Aanes Date: Wed, 8 Jan 2025 15:00:26 +0100 Subject: [PATCH 09/10] hardcode filter for now --- favro_sync/favro_fuse.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/favro_sync/favro_fuse.py b/favro_sync/favro_fuse.py index 1b4ef66..cd17978 100644 --- a/favro_sync/favro_fuse.py +++ b/favro_sync/favro_fuse.py @@ -168,13 +168,16 @@ class FavroFuse(fuse.Fuse): def __init__( self, + *, favro_client: FavroClient, formatter: CardFileFormatter, + collection_filter: frozenset[str], **kwargs, ): self.favro_client = favro_client self.formatter = formatter self.wiped_cards = set() + self.collection_filter = collection_filter self.path_components = [ RootFileSystemItem, CollectionFileSystemItem, @@ -209,6 +212,9 @@ class FavroFuse(fuse.Fuse): return -errno.ENOENT return st + def _is_allowed_collection(self, item: CollectionFileSystemItem) -> bool: + return item.collection_name in self.collection_filter + def readdir(self, path: str, offset: int) -> Iterator[fuse.Direntry]: logger.info('Reading directory: %s', path) file_system_item = path_to_file_system_item(path, self.path_components) @@ -218,11 +224,15 @@ class FavroFuse(fuse.Fuse): if isinstance(file_system_item, RootFileSystemItem): for collection in self.favro_client.get_collections(): - collection_name = collection.name.replace('/', '') - yield fuse.Direntry(str(CollectionFileSystemItem(collection_name))) - del collection + item = CollectionFileSystemItem(collection.name.replace('/', '')) + if self._is_allowed_collection(item): + yield fuse.Direntry(str(item)) + del collection, item elif isinstance(file_system_item, CollectionFileSystemItem): + if not self._is_allowed_collection(file_system_item): + return # Yield nothing + # TODO: move into own function for collection in self.favro_client.get_collections(): if collection.name.replace('/', '') == file_system_item.collection_name: @@ -339,10 +349,12 @@ Userspace hello example def start_favro_fuse(favro_client: FavroClient): logger.info('Starting favro FUSE') + collection_filter = frozenset(['Platform']) # TODO: server = FavroFuse( favro_client=favro_client, formatter=CardFileFormatter(), + collection_filter=collection_filter, version='%prog ' + fuse.__version__, usage=HELP, dash_s_do='setsingle', From a1f4a8207e5b1e77112d407d959cd53d37b67032 Mon Sep 17 00:00:00 2001 From: Jon Michael Aanes Date: Wed, 8 Jan 2025 15:12:57 +0100 Subject: [PATCH 10/10] Load collection filter from file --- favro_sync/__main__.py | 2 +- favro_sync/favro_client.py | 2 +- favro_sync/favro_fuse.py | 8 ++++---- favro_sync/secrets.py | 10 ++++++++++ 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/favro_sync/__main__.py b/favro_sync/__main__.py index b0bef18..7f8de10 100644 --- a/favro_sync/__main__.py +++ b/favro_sync/__main__.py @@ -29,7 +29,7 @@ def main(): ) client.check_logged_in() - start_favro_fuse(client) + start_favro_fuse(client, secrets.favro_collection_filter()) if __name__ == '__main__': diff --git a/favro_sync/favro_client.py b/favro_sync/favro_client.py index ecd9ff7..4614461 100644 --- a/favro_sync/favro_client.py +++ b/favro_sync/favro_client.py @@ -215,7 +215,7 @@ class FavroClient: def _invalidate_cache(self, card_id: CardId) -> None: card = self.card_cache.remove(card_id) if card: - self.session.card_cache.delete( + self.session.cache.delete( requests=[self._get_cards_prepared_request(seq_id=card.seq_id)], ) diff --git a/favro_sync/favro_fuse.py b/favro_sync/favro_fuse.py index cd17978..c2d8a06 100644 --- a/favro_sync/favro_fuse.py +++ b/favro_sync/favro_fuse.py @@ -213,6 +213,8 @@ class FavroFuse(fuse.Fuse): return st def _is_allowed_collection(self, item: CollectionFileSystemItem) -> bool: + if self.collection_filter is None: + return True return item.collection_name in self.collection_filter def readdir(self, path: str, offset: int) -> Iterator[fuse.Direntry]: @@ -347,10 +349,8 @@ Userspace hello example ) -def start_favro_fuse(favro_client: FavroClient): - logger.info('Starting favro FUSE') - collection_filter = frozenset(['Platform']) - # TODO: +def start_favro_fuse(favro_client: FavroClient, collection_filter: frozenset[str]): + logger.info('Starting favro FUSE with collection filter: %s', collection_filter) server = FavroFuse( favro_client=favro_client, formatter=CardFileFormatter(), diff --git a/favro_sync/secrets.py b/favro_sync/secrets.py index 8e45c52..c3fe255 100644 --- a/favro_sync/secrets.py +++ b/favro_sync/secrets.py @@ -17,3 +17,13 @@ def favro_username(): def favro_password(): return secrets.load_or_fail('FAVRO_PASSWORD') + + +def favro_collection_filter() -> frozenset[str]: + loaded = secrets.load('FAVRO_COLLECTION_FILTER') + if loaded is None: + return None + values = loaded.strip().split('\n') + values = [v.strip() for v in values] + values = [v for v in values if v] + return frozenset(values)