from cmdbox.app import common, client
from cmdbox.app.commons import convert, redis_client, resdata, validator
from cmdbox.app.features.cli.rag import rag_base, rag_store
from cmdbox.app.options import Options
from pathlib import Path
from typing import Dict, Any, Tuple, List, Union
import argparse
import logging
import json
import pydantic
import re
[docs]
class RagBuild(rag_base.RAGBase, validator.Validator):
[docs]
def get_mode(self) -> Union[str, List[str]]:
"""
この機能のモードを返します
Returns:
Union[str, List[str]]: モード
"""
return 'rag'
[docs]
def get_cmd(self) -> str:
"""
この機能のコマンドを返します
Returns:
str: コマンド
"""
return 'build'
[docs]
def get_option(self) -> Dict[str, Any]:
"""
この機能のオプションを返します
Returns:
Dict[str, Any]: オプション
"""
return dict(
use_redis=self.USE_REDIS_TRUE, nouse_webmode=False, use_agent=False,
description_ja="RAG(検索拡張生成)の設定を元にデータベースを構築します。",
description_en="We build the database based on the RAG (Retrieval-Augmented Generation) configuration.",
choice=[
dict(opt="host", type=Options.T_STR, default=self.default_host, required=True, multi=False, hide=True, choice=None, web="mask",
description_ja="Redisサーバーのサービスホストを指定します。",
description_en="Specify the service host of the Redis server."),
dict(opt="port", type=Options.T_INT, default=self.default_port, required=True, multi=False, hide=True, choice=None, web="mask",
description_ja="Redisサーバーのサービスポートを指定します。",
description_en="Specify the service port of the Redis server."),
dict(opt="password", type=Options.T_PASSWD, default=self.default_pass, required=True, multi=False, hide=True, choice=None, web="mask",
description_ja=f"Redisサーバーのアクセスパスワード(任意)を指定します。省略時は `{self.default_pass}` を使用します。",
description_en=f"Specify the access password of the Redis server (optional). If omitted, `{self.default_pass}` is used."),
dict(opt="svname", type=Options.T_STR, default=self.default_svname, required=True, multi=False, hide=True, choice=None, web="readonly",
description_ja="サーバーのサービス名を指定します。",
description_en="Specify the service name of the inference server."),
dict(opt="retry_count", type=Options.T_INT, default=3, required=False, multi=False, hide=True, choice=None,
description_ja="Redisサーバーへの再接続回数を指定します。",
description_en="Specifies the number of reconnections to the Redis server."),
dict(opt="retry_interval", type=Options.T_INT, default=5, required=False, multi=False, hide=True, choice=None,
description_ja="Redisサーバーに再接続までの秒数を指定します。",
description_en="Specifies the number of seconds before reconnecting to the Redis server."),
dict(opt="timeout", type=Options.T_INT, default=120, required=False, multi=False, hide=True, choice=None,
description_ja="サーバーの応答が返ってくるまでの最大待ち時間を指定。",
description_en="Specify the maximum waiting time until the server responds."),
dict(opt="rag_name", type=Options.T_STR, default=None, required=True, multi=False, hide=False, choice=None,
description_ja="構築するRAG設定の名前を指定します。",
description_en="Specify the name of the RAG configuration to build."),
]
)
[docs]
@validator.async_apprun_check
async def apprun(self, logger: logging.Logger, args: argparse.Namespace, tm: float, pf: List[Dict[str, float]] = []) -> Tuple[int, Dict[str, Any], Any]:
cl = client.Client(logger, redis_host=args.host, redis_port=args.port, redis_password=args.password, svname=args.svname)
st, rag_conf, _ = await self.load_rag_config(args, cl, tm, pf, logger)
if st != self.RESP_SUCCESS:
return st, rag_conf, cl
payload = dict(rag_conf=rag_conf)
payload_b64 = convert.str2b64str(common.to_str(payload))
ret = await cl.redis_cli.asend_cmd(self.get_svcmd(), [payload_b64],
retry_count=args.retry_count, retry_interval=args.retry_interval, timeout=args.timeout, nowait=False)
common.print_format(ret, args.format, tm, pf=pf)
if 'success' not in ret:
return self.RESP_WARN, ret, cl
return self.RESP_SUCCESS, ret, cl
[docs]
def output_schema(self) -> type:
class Data(resdata.Data):
data: Union[str, None] = pydantic.Field(default=None, description="処理結果のデータ")
class Result(resdata.Result):
success: Union[Data, None] = pydantic.Field(default=None, description="成功した場合の結果")
return Result
[docs]
def is_cluster_redirect(self):
return False
[docs]
def svrun(self, data_dir:Path, logger:logging.Logger, redis_cli:redis_client.RedisClient, msg:List[str],
sessions:Dict[str, Dict[str, Any]]) -> int:
reskey = msg[1]
try:
payload = json.loads(convert.b64str2str(msg[2]))
rag_conf = payload.get('rag_conf')
rag_name = rag_conf.get('rag_name')
embed_vector_dim = rag_conf.get('embed_vector_dim')
ds_conf = self.ds_load.load_datasource(data_dir, rag_conf.get('rag_datasource'))
store = rag_store.RagStore.create(ds_conf, logger,
appcls=self.appcls, ver=self.ver, language=self.language)
store.install()
store.create_tables(rag_name, embed_vector_dim=embed_vector_dim)
msg = dict(success="RAG build completed successfully.")
redis_cli.rpush(reskey, msg)
return self.RESP_SUCCESS
except Exception as e:
msg = dict(warn=f"{self.get_mode()}_{self.get_cmd()}: {e}")
logger.warning(f"{self.get_mode()}_{self.get_cmd()}: {e}", exc_info=True)
redis_cli.rpush(reskey, msg)
return self.RESP_WARN