From a8255f14d8c5729ffd5498c1b64ceb85b17f1b27 Mon Sep 17 00:00:00 2001 From: Alexander Munch-Hansen Date: Tue, 29 Dec 2020 20:20:49 +0100 Subject: [PATCH] meh --- .gitignore | 316 +++++++++++++++++++++++++++++++++++++++++++++++ certificate.py | 36 ++++++ csv_converter.py | 26 ++++ main.py | 56 +++++++++ requirements.txt | 31 +++++ 5 files changed, 465 insertions(+) create mode 100644 .gitignore create mode 100644 certificate.py create mode 100644 csv_converter.py create mode 100644 main.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9462bba --- /dev/null +++ b/.gitignore @@ -0,0 +1,316 @@ + +# Created by https://www.toptal.com/developers/gitignore/api/python,jetbrains,pycharm +# Edit at https://www.toptal.com/developers/gitignore?templates=python,jetbrains,pycharm + +### JetBrains ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### JetBrains Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint +.idea/**/sonarlint/ + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin +.idea/**/sonarIssues.xml + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced +.idea/**/markdown-navigator.xml +.idea/**/markdown-navigator-enh.xml +.idea/**/markdown-navigator/ + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 +.idea/$CACHE_FILE$ + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream +.idea/codestream.xml + +### PyCharm ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff + +# Generated files + +# Sensitive or high-churn files + +# Gradle + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake + +# Mongo Explorer plugin + +# File-based project format + +# IntelliJ + +# mpeltonen/sbt-idea plugin + +# JIRA plugin + +# Cursive Clojure plugin + +# Crashlytics plugin (for Android Studio and IntelliJ) + +# Editor-based Rest Client + +# Android studio 3.1+ serialized cache file + +### PyCharm Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +pytestdebug.log + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +doc/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# profiling data +.prof + +# End of https://www.toptal.com/developers/gitignore/api/python,jetbrains,pycharm diff --git a/certificate.py b/certificate.py new file mode 100644 index 0000000..68da360 --- /dev/null +++ b/certificate.py @@ -0,0 +1,36 @@ +class Certificate(object): + def __init__(self, title, introParsables, exampleParsables, questionParsables, questions): + self.title = title + self.introParsables = introParsables + self.exampleParsables = exampleParsables + self.questionParsables = questionParsables + self.questions = questions + + @staticmethod + def from_dict(source): + title = source['Emne:'] + introParsables = source['Introtekst:'] + exampleParsables = source['Opgaveeksempel:'] + questionParsables = source['Opgavetekst:'] + questions = source['questions'] + return Certificate(title, introParsables, exampleParsables, questionParsables, questions) + + def to_dict(self): + data = {} + data['title'] = self.title + data['introParsables'] = self.introParsables + data['exampleParsables'] = self.exampleParsables + data['questionParsables'] = self.questionParsables + data['questions'] = self.questions + return data + + def __repr__(self): + return( + f'Certificate(\ + title={self.title},\ + intro={self.introParsables},\ + example={self.exampleParsables},\ + questions_intro={self.questionParsables},\ + questions={self.questions}\ + )' + ) diff --git a/csv_converter.py b/csv_converter.py new file mode 100644 index 0000000..45641ee --- /dev/null +++ b/csv_converter.py @@ -0,0 +1,26 @@ +import csv +import json +import codecs +from certificate import Certificate +from io import StringIO + + +def convert(csv_string): + csv_stuff = StringIO(csv_string) + csv_reader = csv.reader(csv_stuff, delimiter=";") + data = {} + for rows in csv_reader: + if rows[0]: + cur_row = rows[0].strip() + if cur_row.find("Svar") != -1: + cur_row = "questions" + if cur_row not in data: + data[cur_row] = [] + if rows[1] != '': + data[cur_row].append(f'text:{rows[1].strip()}') + return Certificate.from_dict(data) + + +if __name__ == "__main__": + cert = convert("opg1_dansk.csv") + print(cert) diff --git a/main.py b/main.py new file mode 100644 index 0000000..98666de --- /dev/null +++ b/main.py @@ -0,0 +1,56 @@ +import os +import firebase_admin +import argparse +from certificate import Certificate +from csv_converter import convert +from firebase_admin import credentials +from firebase_admin import firestore + +parser = argparse.ArgumentParser(description="Rubbish front for google cloud firestore API") +parser.add_argument('--source_file', action='store', dest='source_file', + type=str, required=True, + help='which file to read from') +parser.add_argument('--collection_id', action='store', dest='collection_id', + type=str, require=True, + help='The id of the collection the contents of the source file should be added to') +parser.add_argument('--project_id', action='store', dest='project_id', + type=str, default='navedu-test', + help='The id of the project worked in') + +args = parser.parse_args() + +os.environ["GOOGLE_APPLICATION_CREDENTIALS"]="/Users/alexandermunch-hansen/projects/Python/cloud_firestore/navedu-admin-key.json" + + +def take_huge_file(long_ass_string: str): + with open(long_ass_string, 'r') as file: + bunch_of_shit = file.read() + csv_files = bunch_of_shit.split("$*$") + csv_files = [file for file in csv_files if file != '\ufeff'] + return csv_files + + +def main(source_file, collection_id, project_id): + # Use the application default credentials + cred = credentials.ApplicationDefault() + firebase_admin.initialize_app(cred, { + 'projectId': project_id, + }) + + db = firestore.client() + + csv_strings = take_huge_file(f'{source_file}') + for idx, string in enumerate(csv_strings, start=1): + if idx < 10: + num = f'0{idx}' + else: + num = idx + + certificate = convert(string) + + db.collection(u'certifications').document(f'{collection_id}').collection('subjects').document(f'{num}').set( + certificate.to_dict()) + + +if __name__ == "__main__": + main(args.source_file, args.collection_id, args.project_id) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f460a9d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,31 @@ +CacheControl==0.12.6 +cachetools==4.2.0 +certifi==2020.12.5 +cffi==1.14.4 +chardet==4.0.0 +firebase-admin==4.5.0 +google-api-core==1.24.1 +google-api-python-client==1.12.8 +google-auth==1.24.0 +google-auth-httplib2==0.0.4 +google-cloud-core==1.5.0 +google-cloud-firestore==2.0.2 +google-cloud-storage==1.35.0 +google-crc32c==1.1.0 +google-resumable-media==1.2.0 +googleapis-common-protos==1.52.0 +grpcio==1.34.0 +httplib2==0.18.1 +idna==2.10 +msgpack==1.0.2 +proto-plus==1.13.0 +protobuf==3.14.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycparser==2.20 +pytz==2020.5 +requests==2.25.1 +rsa==4.6 +six==1.15.0 +uritemplate==3.0.1 +urllib3==1.26.2