Last active
February 22, 2022 14:56
-
-
Save ramiroaznar/06c26c95485e4fa2e7666641c4cb5157 to your computer and use it in GitHub Desktop.
Create a dbt project from a project name and a list of data model names
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import os | |
| import click | |
| import logging | |
| import time | |
| # create a logger function | |
| def get_logger(): | |
| logging.basicConfig( | |
| level=logging.INFO, | |
| format=' %(asctime)s - %(name)-18s - %(levelname)-8s %(message)s', | |
| datefmt='%I:%M:%S %p') | |
| logging.Formatter.converter = time.gmtime | |
| logger = logging.getLogger(__name__) | |
| return logger | |
| # create a dbt project with a glossary and a list of folders with sql and yml files | |
| def create_dbt_project(project_name, models_list): | |
| logger = get_logger() | |
| # create a project folder | |
| my_path = os.path.abspath(os.getcwd()) | |
| dir_path = str(my_path) + '/vault-dbt/models/' + project_name | |
| os.mkdir(dir_path) | |
| logger.info('Created project folder: ' + dir_path) | |
| # create a glossary file | |
| glossary_path_name = dir_path + '/' + project_name + '_glossary.md' | |
| title = project_name.replace('_', ' ').title() | |
| with open(glossary_path_name, 'w') as file: | |
| lines = [ | |
| f'# {title} Glossary\n', | |
| '\n', | |
| '## Dashboards\n', | |
| '\n', | |
| '* [Dashboard 1](dashboard1.html)\n', | |
| '\n', | |
| '## Models\n', | |
| '\n' | |
| ] | |
| file.writelines(lines) | |
| logger.info('Created glossary file: ' + glossary_path_name) | |
| for name in models_list: | |
| # create a folder for each model | |
| dir_name = dir_path + '/' + name | |
| os.mkdir(dir_name) | |
| logger.info('Created model folder: ' + dir_name) | |
| # add model name to glossary | |
| with open(glossary_path_name, 'a') as file: | |
| lines = [ | |
| f'### {name}\n', | |
| 'Description\n', | |
| '\n' | |
| ] | |
| file.writelines(lines) | |
| # create a sql file for each model | |
| sql_path_name = dir_name +'/'+ name +'.sql' | |
| with open(sql_path_name, 'w') as file: | |
| lines = [ | |
| "{{ config(materialized='view') }}", | |
| "SELECT * FROM {{ref('model_name')}}" | |
| ] | |
| file.writelines(lines) | |
| logger.info('Created sql file: ' + sql_path_name) | |
| # create a yml file for each model | |
| yml_path_name = dir_name +'/'+ name +'.yml' | |
| with open(yml_path_name, 'w') as file: | |
| lines = [ | |
| 'version: 2\n', | |
| '\n', | |
| 'models:\n', | |
| f' - name: {name}\n', | |
| ' description: Description\n', | |
| '\n', | |
| ' columns:\n', | |
| ' - name: column_name\n', | |
| ' - name: column_name\n', | |
| ' - name: column_name\n', | |
| ' - name: column_name\n', | |
| ' - name: column_name\n', | |
| ' - name: column_name\n', | |
| ' - name: column_name\n', | |
| ' - name: column_name\n' | |
| ] | |
| file.writelines(lines) | |
| logger.info('Created yml file: ' + yml_path_name) | |
| @click.command() | |
| @click.option( | |
| '--project_name', | |
| '-n', | |
| help='Name of the project') | |
| @click.option( | |
| '--model_name', | |
| '-m', | |
| multiple=True, | |
| help='List of model names') | |
| @click.help_option('-h', '--help') | |
| def main(project_name, model_name): | |
| logger = get_logger() | |
| try: | |
| create_dbt_project(project_name, model_name) | |
| except Exception as e: | |
| logger.info('Error: ' + str(e)) | |
| if __name__ == "__main__": | |
| main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment