Skip to content

cianfhoghlaim/cianchosaint / .agents/skills/cocoindex

cocoindex

Verified

references/patterns.md

View on GitHub
# CocoIndex Common Patterns

Common patterns and workflows for building CocoIndex pipelines.

## Core Pattern: TargetState = Transform(SourceState)

All CocoIndex pipelines follow this declarative pattern:

1. Read source state
2. Transform
3. Declare target state

CocoIndex handles the incremental sync automatically.

---

## Pattern 1: File Transformation Pipeline

**Use case**: Transform files from one format to another (e.g., Markdown -> HTML, PDF -> Markdown)

```python
import pathlib
import cocoindex as coco
from cocoindex.connectors import localfs
from cocoindex.resources.file import FileLike, PatternFilePathMatcher
from markdown_it import MarkdownIt

_markdown_it = MarkdownIt("gfm-like")

@coco.fn(memo=True)
async def process_file(file: FileLike, outdir: pathlib.Path) -> None:
    html = _markdown_it.render(await file.read_text())
    outname = "__".join(file.file_path.path.parts) + ".html"
    localfs.declare_file(outdir / outname, html, create_parent_dirs=True)

@coco.fn
async def app_main(sourcedir: pathlib.Path, outdir: pathlib.Path) -> None:
    files = localfs.walk_dir(
        sourcedir,
        path_matcher=PatternFilePathMatcher(included_patterns=["**/*.md"]),
        live=True,  # Enable live file watching
    )
    await coco.mount_each(process_file, files.items(), outdir)

app = coco.App(
    coco.AppConfig(name="FilesTransform"),
    app_main,
    sourcedir=pathlib.Path("./data"),
    outdir=pathlib.Path("./output_html"),
)
```

**Key points:**
- `memo=True` -- Skip reprocessing unchanged files
- `mount_each()` -- One component per file; keys from `files.items()` become component subpaths
- `live=True` on `walk_dir()` -- Enables file watching for live mode
- Auto-cleanup -- Deleting source file automatically removes output file

---

## Pattern 2: Vector Embedding Pipeline

**Use case**: Chunk and embed documents for semantic search

```python
import pathlib
from dataclasses import dataclass
from typing import AsyncIterator, Annotated

import asyncpg
from numpy.typing import NDArray

import cocoindex as coco
from cocoindex.connectors import localfs, postgres
from cocoindex.ops.text import RecursiveSplitter
from cocoindex.ops.sentence_transformers import SentenceTransformerEmbedder
from cocoindex.resources.chunk import Chunk
from cocoindex.resources.file import FileLike, PatternFilePathMatcher
from cocoindex.resources.id import IdGenerator

DATABASE_URL = "postgres://cocoindex:cocoindex@localhost/cocoindex"
PG_DB = coco.ContextKey[asyncpg.Pool]("pg_db")
EMBEDDER = coco.ContextKey[SentenceTransformerEmbedder]("embedder")

_splitter = RecursiveSplitter()

@dataclass
class DocEmbedding:
    id: int
    filename: str
    chunk_start: int
    chunk_end: int
    text: str
    embedding: Annotated[NDArray, EMBEDDER]

@coco.lifespan
async def coco_lifespan(builder: coco.EnvironmentBuilder) -> AsyncIterator[None]:
    async with await asyncpg.create_pool(DATABASE_URL) as pool:
        builder.provide(PG_DB, pool)
        builder.provide(EMBEDDER, SentenceTransformerEmbedder("all-MiniLM-L6-v2"))
        yield

@coco.fn
async def process_chunk(
    chunk: Chunk, filename: pathlib.PurePath,
    id_gen: IdGenerator, table: postgres.TableTarget[DocEmbedding],
) -> None:
    table.declare_row(row=DocEmbedding(
        id=await id_gen.next_id(chunk.text),
        filename=str(filename),
        chunk_start=chunk.start.char_offset,
        chunk_end=chunk.end.char_offset,
        text=chunk.text,
        embedding=await coco.use_context(EMBEDDER).embed(chunk.text),
    ))

@coco.fn(memo=True)
async def process_file(file: FileLike, table: postgres.TableTarget[DocEmbedding]) -> None:
    text = await file.read_text()
    chunks = _splitter.split(text, chunk_size=2000, chunk_overlap=500, language="markdown")
    id_gen = IdGenerator()
    await coco.map(process_chunk, chunks, file.file_path.path, id_gen, table)

@coco.fn
async def app_main(sourcedir: pathlib.Path) -> None:
    target_table = await postgres.mount_table_target(
        PG_DB,
        table_name="doc_embeddings",
        table_schema=await postgres.TableSchema.from_class(DocEmbedding, primary_key=["id"]),
    )
    target_table.declare_vector_index(column="embedding")

    files = localfs.walk_dir(sourcedir, recursive=True,
        path_matcher=PatternFilePathMatcher(included_patterns=["**/*.md"]))
    await coco.mount_each(process_file, files.items(), target_table)

app = coco.App(coco.AppConfig(name="TextEmbedding"), app_main,
    sourcedir=pathlib.Path("./markdown_files"))
```

**Key points:**
- `mount_table_target(PG_DB, ...)` -- Takes `ContextKey` as first argument
- `Annotated[NDArray, EMBEDDER]` -- Vector annotation uses ContextKey for auto dimension inference
- `map()` -- Concurrent execution within a component (no child components created)
- `IdGenerator` -- Generates stable unique IDs for chunks across incremental updates
- `memo=True` on `process_file` -- Skips unchanged files entirely

---

## Pattern 3: Database Source -> Transform -> Database Target

**Use case**: Transform data from one database to another

```python
from dataclasses import dataclass
from typing import AsyncIterator

import asyncpg
import cocoindex as coco
from cocoindex.connectors import postgres

SOURCE_DB_URL = "postgres://localhost/source_db"
TARGET_DB_URL = "postgres://localhost/target_db"

SOURCE_DB = coco.ContextKey[asyncpg.Pool]("source_db")
TARGET_DB = coco.ContextKey[asyncpg.Pool]("target_db")

@dataclass
class SourceRecord:
    id: int
    name: str
    value: float

@dataclass
class TargetRecord:
    id: int
    name: str
    value: float
    processed: bool

@coco.lifespan
async def coco_lifespan(builder: coco.EnvironmentBuilder) -> AsyncIterator[None]:
    async with (
        await asyncpg.create_pool(SOURCE_DB_URL) as source_pool,
        await asyncpg.create_pool(TARGET_DB_URL) as target_pool,
    ):
        builder.provide(SOURCE_DB, source_pool)
        builder.provide(TARGET_DB, target_pool)
        yield

@coco.fn(memo=True)
async def process_record(record: SourceRecord, target_table: postgres.TableTarget[TargetRecord]) -> None:
    target_table.declare_row(row=TargetRecord(
        id=record.id, name=record.name.upper(),
        value=record.value * 2, processed=True,
    ))

@coco.fn
async def app_main() -> None:
    target_table = await postgres.mount_table_target(
        TARGET_DB,
        table_name="target_records",
        table_schema=await postgres.TableSchema.from_class(TargetRecord, primary_key=["id"]),
    )

    source = postgres.PgTableSource(
        coco.use_context(SOURCE_DB),
        table_name="source_records",
        row_type=SourceRecord,
    )
    await coco.mount_each(
        coco.component_subpath("record"),
        process_record,
        source.fetch_rows().items(key=lambda r: r.id),
        target_table,
    )

app = coco.App(coco.AppConfig(name="DatabaseTransform"), app_main)
```

---

## Pattern 4: LLM-Based Extraction Pipeline

**Use case**: Extract structured data using LLMs

```python
import instructor
from dataclasses import dataclass
from typing import AsyncIterator
from pydantic import BaseModel
from litellm import acompletion

import asyncpg
import cocoindex as coco
from cocoindex.connectors import postgres

DATABASE_URL = "postgres://cocoindex:cocoindex@localhost/cocoindex"
PG_DB = coco.ContextKey[asyncpg.Pool]("pg_db")

_instructor_client = instructor.from_litellm(acompletion, mode=instructor.Mode.JSON)

class ExtractedTopic(BaseModel):
    name: str
    description: str

class ExtractionResult(BaseModel):
    title: str
    topics: list[ExtractedTopic]

@dataclass
class Message:
    id: int
    title: str
    content: str

@dataclass
class Topic:
    message_id: int
    name: str
    description: str

@coco.fn(memo=True)
async def extract_and_store(
    content: str, message_id: int,
    messages_table: postgres.TableTarget[Message],
    topics_table: postgres.TableTarget[Topic],
) -> None:
    result = await _instructor_client.chat.completions.create(
        model="gpt-4",
        response_model=ExtractionResult,
        messages=[{"role": "user", "content": f"Extract topics:\n\n{content}"}],
    )
    messages_table.declare_row(row=Message(id=message_id, title=result.title, content=content))
    for topic in result.topics:
        topics_table.declare_row(row=Topic(
            message_id=message_id, name=topic.name, description=topic.description,
        ))

@coco.fn
async def app_main(input_texts: list[str]) -> None:
    messages_table = await postgres.mount_table_target(
        PG_DB, table_name="messages",
        table_schema=await postgres.TableSchema.from_class(Message, primary_key=["id"]),
    )
    topics_table = await postgres.mount_table_target(
        PG_DB, table_name="topics",
        table_schema=await postgres.TableSchema.from_class(Topic, primary_key=["message_id", "name"]),
    )
    for idx, text in enumerate(input_texts):
        await coco.mount(
            coco.component_subpath("text", idx),
            extract_and_store, text, idx, messages_table, topics_table,
        )

app = coco.App(coco.AppConfig(name="LLMExtraction"), app_main, input_texts=["text1...", "text2..."])
```

**Key points:**
- `memo=True` on extraction -- Avoids re-calling LLM for unchanged inputs
- Multiple target tables -- Declare rows in multiple tables from single component
- Pydantic models -- For structured LLM outputs

---

## Pattern 5: Kafka Streaming Pipeline

**Use case**: Process Kafka messages and write to a database

```python
import json
from collections.abc import AsyncIterator
from dataclasses import dataclass

from confluent_kafka import Message
from confluent_kafka.aio import AIOConsumer

import cocoindex as coco
from cocoindex.connectors import kafka, lancedb

LANCE_DB = coco.ContextKey[lancedb.LanceAsyncConnection]("lance_db")

@dataclass
class Product:
    sku: str
    name: str
    category: str
    price: float

@coco.lifespan
async def coco_lifespan(builder: coco.EnvironmentBuilder) -> AsyncIterator[None]:
    conn = await lancedb.connect_async("./lancedb_data")
    builder.provide(LANCE_DB, conn)
    yield

@coco.fn
async def process_message(msg: Message, table: lancedb.TableTarget[Product]) -> None:
    value = msg.value()
    if value is None:
        return
    row = json.loads(value.decode() if isinstance(value, bytes) else value)
    table.declare_row(row=Product(**{**row, "price": float(row["price"])}))

@coco.fn
async def app_main() -> None:
    products_table = await lancedb.mount_table_target(
        LANCE_DB, table_name="products",
        table_schema=await lancedb.TableSchema.from_class(Product, primary_key=["sku"]),
    )

    consumer = AIOConsumer({
        "bootstrap.servers": "localhost:9092",
        "group.id": "my-group",
        "enable.auto.commit": "false",
        "auto.offset.reset": "earliest",
    })
    items = kafka.topic_as_map(consumer, ["products-topic"])
    await coco.mount_each(process_message, items, products_table)

app = coco.App(coco.AppConfig(name="KafkaToLanceDB"), app_main)
```

**Key points:**
- `kafka.topic_as_map()` returns a `LiveMapFeed` -- `mount_each()` auto-detects for live mode
- Pipeline runs continuously in live mode, processing new messages as they arrive

---

## Pattern 6: Context Management for Shared Resources

**Use case**: Share expensive resources across components

```python
import cocoindex as coco
from cocoindex.ops.sentence_transformers import SentenceTransformerEmbedder

EMBEDDER = coco.ContextKey[SentenceTransformerEmbedder]("embedder")
CONFIG = coco.ContextKey[dict]("config")

@coco.lifespan
async def coco_lifespan(builder: coco.EnvironmentBuilder) -> AsyncIterator[None]:
    builder.provide(EMBEDDER, SentenceTransformerEmbedder("all-MiniLM-L6-v2"))
    builder.provide(CONFIG, {"chunk_size": 1000, "overlap": 200})
    yield

@coco.fn
async def process_item(text: str) -> None:
    embedder = coco.use_context(EMBEDDER)
    config = coco.use_context(CONFIG)
    embedding = await embedder.embed(text)
    ...
```

**Key points:**
- `detect_change=True` -- Opt in to invalidate memos when value changes (use for models/config affecting output)
- `detect_change=False` (default) -- For resources not affecting computation (DB connections, loggers)
- `ContextKey` name is stable identity -- avoid renaming across runs

---

## Common Anti-Patterns to Avoid

### Missing `@coco.fn` Decorator

```python
# BAD: Missing decorator
async def process_file(file, table):
    table.declare_row(...)

# GOOD:
@coco.fn
async def process_file(file, table):
    table.declare_row(...)
```

### Reprocessing Everything

```python
# BAD: No memoization
@coco.fn  # Missing memo=True
async def process_file(file, table):
    embedding = await embedder.embed(await file.read_text())  # Expensive!

# GOOD:
@coco.fn(memo=True)
async def process_file(file, table):
    ...
```

### Unstable Component Paths

```python
# BAD: Using object references or indices
for file in files:
    await coco.mount(coco.component_subpath(file), ...)     # Object ref
for idx, item in enumerate(items):
    await coco.mount(coco.component_subpath(idx), ...)      # Index changes

# GOOD: Use stable identifiers
await coco.mount_each(process_file, files.items(), table)   # Keys from items()
```

### Loading Resources Per Component

```python
# BAD: Loading model in every component
@coco.fn
async def process(text):
    model = SentenceTransformer("model")  # Loaded repeatedly!

# GOOD: Load once in lifespan, use via context
embedder = coco.use_context(EMBEDDER)
```

### Mixing Target State with Side Effects

```python
# BAD: Side effects not detected
@coco.fn
async def process(data):
    requests.post("https://api.example.com", json=data)  # Not detected!

# GOOD: Only declare target states
table.declare_row(row=result)
```

---

## See Also

- [Connectors Reference](./connectors.md)
- [API Reference](./api_reference.md)
- [Setup Project](./setup_project.md)
slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack."Stars: 198kUpdated Claude CodeCodexVerifiedInstall
ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report.Stars: 194kUpdated Claude CodeCodexVerifiedInstall
ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt.Stars: 194kUpdated Claude CodeCodexVerifiedInstall