Feature/access for rdb resource

.github/workflows/install-lint-test-on-mac.yml

@@ -15,10 +15,25 @@ jobs:
 
     strategy:
       matrix:
-        python-version:  # github.com/actions/python-versions/releases
+        python-version: # github.com/actions/python-versions/releases
           - 3.12
           # - 3.13
 
+    services:
+      mysql:
+        image: mysql:9.1.0
+        env:
+          MYSQL_USER: root
+          MYSQL_PASSWORD: password
+          MYSQL_DATABASE: test
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping --silent"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4  # github.com/actions/checkout
@@ -28,12 +43,22 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
       - name: Install Poetry
         run: make get-poetry
 
       - name: Install Package & Dependencies
         run: make install
 
+      - name: Wait for MySQL to be ready
+        run: |
+          until mysqladmin ping -h 127.0.0.1 --silent; do
+            echo "Waiting for MySQL..."
+            sleep 5
+          done
+
       - name: Lint Code
         run: make lint
 
@@ -42,3 +67,6 @@ jobs:
         env:
           LEPTON_API_KEY: ${{ secrets.LEPTON_API_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          MYSQL_USER: root
+          MYSQL_PASSWORD: password
+          MYSQL_DATABASE: test

.github/workflows/install-lint-test-on-ubuntu.yml

@@ -19,6 +19,20 @@ jobs:
           - 3.12
           # - 3.13
 
+    services:
+      mysql:
+        image: mysql:latest
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+          MYSQL_DATABASE: test
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping --silent"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4  # github.com/actions/checkout
@@ -34,6 +48,22 @@ jobs:
       - name: Install Package & Dependencies
         run: make install
 
+      - name: Wait for MySQL to be ready
+        run: |
+          until mysqladmin ping -h 127.0.0.1 --silent; do
+            echo "Waiting for MySQL..."
+            sleep 5
+          done
+
+      - name: Install Alembic
+        run: pip install alembic
+
+      - name: Run Migrations
+        run: |
+          python -m alembic upgrade head
+        env:
+          DATABASE_URL: mysql+pymysql://[email protected]/test
+
       - name: Lint Code
         run: make lint
 
@@ -42,3 +72,4 @@ jobs:
         env:
           LEPTON_API_KEY: ${{ secrets.LEPTON_API_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          DB_NAME: test

alembic.ini

@@ -0,0 +1,85 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# timezone to use when rendering the date
+# within the migration file as well as the filename.
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = mysql+pymysql://user:pass@localhost/dbname
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks=black
+# black.type=console_scripts
+# black.entrypoint=black
+# black.options=-l 79
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

examples/japanese-easy-demo/pyproject.toml

@@ -8,6 +8,7 @@ readme = "README.md"
 [tool.poetry.dependencies]
 python = ">=3.12,<3.13"
 openssa = "^0.24.3.12"
+python-dotenv = "^1.0.1"
 
 
 [build-system]

examples/use-rdb-resource/.env.template

@@ -0,0 +1,7 @@
+OPENAI_API_KEY=your openai API key
+
+DB_USERNAME=your_username
+DB_PASSWORD=your_password
+DB_HOST=your_host
+DB_PORT=your_port
+DB_NAME=your_database_name

examples/use-rdb-resource/.gitignore

@@ -0,0 +1,4 @@
+db_config.yaml
+Chinook.sqlite
+chroma.sqlite3
+*.bin

examples/use-rdb-resource/README.md

@@ -0,0 +1,26 @@
+# Access Database resources and inference by DANA
+
+## What is this example doing?
+
+- Use DbResource file to get data from RDB(MySQL) Using SQL made by Vanna, and answer question by DANA.
+
+## Setting-up
+
+- What you need
+  - commands (if you are mac user, you can install those things by Homebrew)
+    - mysql
+      - Also, create or use existing database for this example.
+    - poetry
+  - API Key
+    - Use your own OpenAI API key
+
+- Setting up Commands
+  - `cd examples/use-rdb-resource`
+  - `poetry install`
+  - `cp .env.template .env`
+    - update .env data with your environment data
+  - `poetry run python make_example_table_data.py`
+    - if this command doesn't work, then run `poetry env use 3.12`
+      - change python version to resolve dependensies version
+  - `poetry run python main.py`
+    - run main file to answer question by DANA using DbResource, and see the result in the terminal.

examples/use-rdb-resource/main.py

@@ -0,0 +1,41 @@
+from dotenv import load_dotenv
+
+from myvanna import generate_sql_from_prompt
+
+# from openssa import DANA, DbResource
+from openssa.core.agent.dana import DANA  # , FileResource
+from openssa.core.resource.db import DbResource
+
+load_dotenv()
+
+
+def get_or_create_agent(query) -> DANA:
+    return DANA(
+        resources=[DbResource(query=query)]
+    )
+
+
+def solve(question, query) -> str:
+    agent = get_or_create_agent(query)
+    try:
+        return agent.solve(problem=question)
+    except ValueError as err:
+        return f'ERROR: {err}'
+    except RuntimeError as err:
+        return f'ERROR: {err}'
+
+
+if __name__ == '__main__':
+    QUESTION = (
+        "Can you list the products in order of sales volume from highest to lowest?"
+    )
+
+    query = generate_sql_from_prompt(QUESTION)
+    print(query)
+    answer = solve(QUESTION, query)
+
+    print('--------------------------------')
+    print(answer)
+    print('--------------------------------')
+    print(query)
+    print('--------------------------------')

examples/use-rdb-resource/make_example_table_data.py

@@ -0,0 +1,103 @@
+import os
+import random
+import secrets
+
+from dotenv import load_dotenv
+from faker import Faker
+from sqlalchemy import Column, Integer, String, Date, inspect, create_engine
+from sqlalchemy.orm import sessionmaker, declarative_base
+
+from myvanna import train_vanna_for_sales_data
+
+load_dotenv()
+
+Base = declarative_base()
+
+
+class SalesData(Base):
+    __tablename__ = 'sales_data'
+    sale_id = Column(Integer, primary_key=True, autoincrement=True)
+    product_id = Column(Integer)
+    product_name = Column(String(255))
+    sale_date = Column(Date)
+    region = Column(String(255))
+
+
+class MySQLDatabase:
+    def __init__(self):
+        self.engine = self.create_engine()
+        self.Session = sessionmaker(bind=self.engine)
+
+    def create_engine(self):
+        username = os.getenv('DB_USERNAME')
+        password = os.getenv('DB_PASSWORD')
+        host = os.getenv('DB_HOST')
+        port = os.getenv('DB_PORT')
+        database = os.getenv('DB_NAME')
+        connection_string = f'mysql+pymysql://{username}:{password}@{host}:{port}/{database}'
+        return create_engine(connection_string)
+
+    def get_session(self):
+        return self.Session()
+
+    def create_tables(self):
+        Base.metadata.create_all(self.engine)
+
+    def drop_table(self, table_class):
+        inspector = inspect(self.engine)
+        if inspector.has_table(table_class.__tablename__):
+            table_class.__table__.drop(self.engine)
+
+
+fake = Faker()
+seed_value = 42
+random.seed(seed_value)
+Faker.seed(seed_value)
+
+products = [
+    {"id": 101, "name": "Smartwatch", "price": 150.00},
+    {"id": 102, "name": "Laptop", "price": 1200.00},
+    {"id": 103, "name": "Smartphone", "price": 800.00},
+    {"id": 104, "name": "Tablet", "price": 400.00},
+    {"id": 105, "name": "Headphones", "price": 100.00}
+]
+
+regions = ["North America", "Europe", "Asia", "South America", "Africa"]
+
+
+def generate_sales_data(session, num_records):
+    sales_data_list = []
+    for _ in range(num_records):
+        product = secrets.choice(products)
+        region = secrets.choice(regions)
+        sale_date = fake.date_between(start_date='-1y', end_date='today')
+        sales_data = SalesData(
+            product_id=product["id"],
+            product_name=product["name"],
+            sale_date=sale_date,
+            region=region
+        )
+        sales_data_list.append(sales_data)
+    session.bulk_save_objects(sales_data_list)
+    session.commit()
+
+
+if __name__ == "__main__":
+    db = MySQLDatabase()
+
+    db.drop_table(SalesData)
+    db.create_tables()
+
+    session = db.get_session()
+
+    generate_sales_data(session, 20000)
+
+    train_vanna_for_sales_data("""
+        CREATE TABLE sales_data (
+            sale_id INT PRIMARY KEY AUTO_INCREMENT,
+            product_id INT,
+            product_name VARCHAR(255),
+            sale_date DATE,
+            region VARCHAR(255)
+        )
+    """)