nexu-io/open-design / design-templates/last30days
last30days
FlaggedRecent community and social trend research over the last 30 days. Use when the brief asks what people are saying now, recent sentiment, community reactions, social proof, launch reaction, trend scan, or last-30-days context.
scripts/lib/tiktok.py
View on GitHub"""TikTok search via ScrapeCreators API for /last30days.
Uses ScrapeCreators REST API to search TikTok by keyword, extract engagement
metrics (views, likes, comments, shares), and fetch video transcripts.
Requires SCRAPECREATORS_API_KEY in config. 100 free API calls, then PAYG.
API docs: https://scrapecreators.com/docs
"""
import re
import sys
from typing import Any, Dict, List, Optional, Set
try:
import requests as _requests
except ImportError:
_requests = None
from . import dates, http, log
SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/tiktok"
# Depth configurations: how many results to fetch / captions to extract
DEPTH_CONFIG = {
"quick": {"results_per_page": 10, "max_captions": 3},
"default": {"results_per_page": 20, "max_captions": 5},
"deep": {"results_per_page": 40, "max_captions": 8},
}
# Max words to keep from each caption
CAPTION_MAX_WORDS = 500
from .relevance import token_overlap_relevance as _compute_relevance
def _extract_core_subject(topic: str) -> str:
"""Extract core subject from verbose query for TikTok search."""
from .query import extract_core_subject
_TIKTOK_NOISE = frozenset({
'best', 'top', 'good', 'great', 'awesome', 'killer',
'latest', 'new', 'news', 'update', 'updates',
'trending', 'hottest', 'popular', 'viral',
'practices', 'features',
'recommendations', 'advice',
'prompt', 'prompts', 'prompting',
'methods', 'strategies', 'approaches',
})
return extract_core_subject(topic, noise=_TIKTOK_NOISE)
def _infer_query_intent(topic: str) -> str:
"""Tiny local intent classifier for TikTok query expansion."""
text = topic.lower().strip()
if re.search(r"\b(vs|versus|compare|difference between)\b", text):
return "comparison"
if re.search(r"\b(how to|tutorial|guide|setup|step by step|deploy|install)\b", text):
return "how_to"
if re.search(r"\b(thoughts on|worth it|should i|opinion|review)\b", text):
return "opinion"
if re.search(r"\b(pricing|feature|features|best .* for)\b", text):
return "product"
return "breaking_news"
def expand_tiktok_queries(topic: str, depth: str) -> List[str]:
"""Generate multiple TikTok search queries from a topic.
Mirrors reddit.py's expand_reddit_queries() pattern:
1. Extract core subject (strip noise words)
2. Include original topic if different from core
3. Add intent-specific OR-joined content-type variants
4. Cap by depth: 1 for quick, 2 for default, 3 for deep
Returns 1-3 query strings depending on depth.
"""
core = _extract_core_subject(topic)
queries = [core]
# Include cleaned original topic as variant if different from core
original_clean = topic.strip().rstrip('?!.')
if core.lower() != original_clean.lower() and len(original_clean.split()) <= 8:
queries.append(original_clean)
qtype = _infer_query_intent(topic)
# Intent-specific TikTok content-type variants
if qtype in ("breaking_news", "opinion"):
queries.append(f"{core} edit OR reaction OR trend")
elif qtype == "product":
queries.append(f"{core} review OR haul OR unboxing")
elif qtype == "comparison":
queries.append(f"{core} vs OR compared OR which is better")
elif qtype == "how_to":
queries.append(f"{core} tutorial OR hack OR tip")
else:
queries.append(f"{core} edit OR reaction OR trend")
# Deep depth: add viral content variant
if depth == "deep":
queries.append(f"{core} viral OR fyp OR trending")
# Cap by depth budget
caps = {"quick": 1, "default": 2, "deep": 3}
cap = caps.get(depth, 2)
return queries[:cap]
def _log(msg: str):
log.source_log("TikTok", msg)
def _parse_date(item: Dict[str, Any]) -> Optional[str]:
"""Parse date from ScrapeCreators TikTok item to YYYY-MM-DD."""
ts = item.get("create_time")
if ts:
try:
return dates.timestamp_to_date(int(ts))
except (ValueError, TypeError):
pass
return None
def _clean_webvtt(text: str) -> str:
"""Strip WebVTT timestamps and headers from transcript text."""
if not text:
return ""
lines = text.split('\n')
cleaned = []
for line in lines:
line = line.strip()
if not line:
continue
if line.startswith('WEBVTT'):
continue
if re.match(r'^\d{2}:\d{2}', line):
continue
if '-->' in line:
continue
cleaned.append(line)
return ' '.join(cleaned)
def _parse_items(raw_items: List[Dict[str, Any]], core_topic: str) -> List[Dict[str, Any]]:
"""Parse raw TikTok items into normalized dicts."""
items = []
for raw in raw_items:
video_id = str(raw.get("aweme_id", ""))
text = raw.get("desc", "")
stats = raw.get("statistics") if isinstance(raw.get("statistics"), dict) else {}
play_count = stats.get("play_count") if stats.get("play_count") is not None else 0
digg_count = stats.get("digg_count") if stats.get("digg_count") is not None else 0
comment_count = stats.get("comment_count") if stats.get("comment_count") is not None else 0
share_count = stats.get("share_count") if stats.get("share_count") is not None else 0
author_raw = raw.get("author")
if isinstance(author_raw, dict):
author_name = author_raw.get("unique_id", "")
elif isinstance(author_raw, str):
author_name = author_raw
else:
author_name = ""
share_url = raw.get("share_url", "")
text_extra = raw.get("text_extra") or []
hashtag_names = [t.get("hashtag_name", "") for t in text_extra
if isinstance(t, dict) and t.get("hashtag_name")]
video_raw = raw.get("video")
duration = video_raw.get("duration") if isinstance(video_raw, dict) else None
date_str = _parse_date(raw)
# Compute relevance with hashtag boost
relevance = _compute_relevance(core_topic, text, hashtag_names)
# Build URL: prefer share_url, fallback to constructed URL
url = share_url.split("?")[0] if share_url else ""
if not url and author_name and video_id:
url = f"https://www.tiktok.com/@{author_name}/video/{video_id}"
items.append({
"video_id": video_id,
"text": text,
"url": url,
"author_name": author_name,
"date": date_str,
"engagement": {
"views": play_count,
"likes": digg_count,
"comments": comment_count,
"shares": share_count,
},
"hashtags": hashtag_names,
"duration": duration,
"relevance": relevance,
"why_relevant": f"TikTok: {text[:60]}" if text else f"TikTok: {core_topic}",
"caption_snippet": "", # populated by fetch_captions
})
return items
def _hashtag_search(
hashtag: str,
token: str,
) -> List[Dict[str, Any]]:
"""Search TikTok by hashtag via ScrapeCreators.
Args:
hashtag: Hashtag name (without #)
token: ScrapeCreators API key
Returns:
List of raw TikTok item dicts (aweme_info format).
"""
_log(f"Hashtag search: #{hashtag}")
if not _requests:
try:
from urllib.parse import urlencode
params = urlencode({"hashtag": hashtag})
url = f"{SCRAPECREATORS_BASE}/search/hashtag?{params}"
headers = http.scrapecreators_headers(token)
headers["User-Agent"] = http.USER_AGENT
data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as e:
_log(f"Hashtag search error (urllib) for #{hashtag}: {e}")
return []
else:
try:
resp = _requests.get(
f"{SCRAPECREATORS_BASE}/search/hashtag",
params={"hashtag": hashtag},
headers=http.scrapecreators_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"Hashtag search error for #{hashtag}: {e}")
return []
raw_items = data.get("aweme_list") or data.get("data") or []
_log(f" -> {len(raw_items)} results for #{hashtag}")
return raw_items
def _profile_videos(
handle: str,
token: str,
count: int = 10,
) -> List[Dict[str, Any]]:
"""Fetch a TikTok creator's recent videos via ScrapeCreators.
Args:
handle: TikTok username (without @)
token: ScrapeCreators API key
count: Max videos to return
Returns:
List of raw TikTok item dicts (aweme_info format).
"""
_log(f"Profile videos: @{handle}")
profile_url = "https://api.scrapecreators.com/v3/tiktok/profile/videos"
if not _requests:
try:
from urllib.parse import urlencode
params = urlencode({"handle": handle, "sort_by": "latest"})
url = f"{profile_url}?{params}"
headers = http.scrapecreators_headers(token)
headers["User-Agent"] = http.USER_AGENT
data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as e:
_log(f"Profile videos error (urllib) for @{handle}: {e}")
return []
else:
try:
resp = _requests.get(
profile_url,
params={"handle": handle, "sort_by": "latest"},
headers=http.scrapecreators_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"Profile videos error for @{handle}: {e}")
return []
raw_items = data.get("aweme_list") or data.get("data") or []
_log(f" -> {len(raw_items)} videos from @{handle}")
return raw_items[:count]
def search_tiktok(
topic: str,
from_date: str,
to_date: str,
depth: str = "default",
token: str = None,
) -> Dict[str, Any]:
"""Search TikTok via ScrapeCreators API.
Args:
topic: Search topic
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
depth: 'quick', 'default', or 'deep'
token: ScrapeCreators API key
Returns:
Dict with 'items' list and optional 'error'.
"""
if not token:
return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
core_topic = _extract_core_subject(topic)
_log(f"Searching TikTok for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
if not _requests:
_log("requests library not installed, falling back to urllib")
try:
from urllib.parse import urlencode
params = urlencode({"query": core_topic, "sort_by": "relevance"})
url = f"{SCRAPECREATORS_BASE}/search/keyword?{params}"
headers = http.scrapecreators_headers(token)
headers["User-Agent"] = http.USER_AGENT
data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as e:
_log(f"ScrapeCreators error (urllib): {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
else:
try:
resp = _requests.get(
f"{SCRAPECREATORS_BASE}/search/keyword",
params={"query": core_topic, "sort_by": "relevance"},
headers=http.scrapecreators_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as e:
_log(f"ScrapeCreators error: {e}")
return {"items": [], "error": f"{type(e).__name__}: {e}"}
# Items are nested under aweme_info
raw_entries = data.get("search_item_list") or data.get("data") or []
raw_items = []
for entry in raw_entries:
if isinstance(entry, dict):
info = entry.get("aweme_info", entry)
raw_items.append(info)
# Limit to configured count
raw_items = raw_items[:config["results_per_page"]]
# Parse items
items = _parse_items(raw_items, core_topic)
# Hard date filter
in_range = [i for i in items if i["date"] and from_date <= i["date"] <= to_date]
out_of_range = len(items) - len(in_range)
if in_range:
items = in_range
if out_of_range:
_log(f"Filtered {out_of_range} videos outside date range")
else:
_log(f"No videos within date range, keeping all {len(items)}")
# Sort by views descending
items.sort(key=lambda x: x["engagement"]["views"], reverse=True)
_log(f"Found {len(items)} TikTok videos")
return {"items": items}
def fetch_captions(
video_items: List[Dict[str, Any]],
token: str,
depth: str = "default",
) -> Dict[str, str]:
"""Fetch transcripts for top N TikTok videos via ScrapeCreators.
Strategy:
1. Use the 'text' field (video description) as baseline caption
2. For top N, call /video/transcript for spoken-word captions
Args:
video_items: Items from search_tiktok()
token: ScrapeCreators API key
depth: Depth level for caption limit
Returns:
Dict mapping video_id -> caption text (truncated to 500 words)
"""
config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
max_captions = config["max_captions"]
if not video_items or not token or not _requests:
return {}
top_items = video_items[:max_captions]
_log(f"Enriching captions for {len(top_items)} videos")
captions = {}
# First pass: use text field as caption (always available, free)
for item in top_items:
vid = item["video_id"]
text = item.get("text", "")
if text:
words = text.split()
if len(words) > CAPTION_MAX_WORDS:
text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
captions[vid] = text
# Second pass: try to get spoken-word transcripts (1 credit each)
for item in top_items:
vid = item["video_id"]
url = item.get("url", "")
if not url:
continue
try:
resp = _requests.get(
f"{SCRAPECREATORS_BASE}/video/transcript",
params={"url": url},
headers=http.scrapecreators_headers(token),
timeout=15,
)
if resp.status_code == 200:
data = resp.json()
transcript = data.get("transcript")
if transcript:
if isinstance(transcript, list):
transcript = " ".join(str(s) for s in transcript)
transcript = _clean_webvtt(transcript)
if transcript:
words = transcript.split()
if len(words) > CAPTION_MAX_WORDS:
transcript = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
captions[vid] = transcript
except Exception as e:
_log(f"Transcript fetch failed for {vid}: {e}")
got = sum(1 for v in captions.values() if v)
_log(f"Got captions for {got}/{len(top_items)} videos")
return captions
def search_and_enrich(
topic: str,
from_date: str,
to_date: str,
depth: str = "default",
token: str = None,
hashtags: List[str] | None = None,
creators: List[str] | None = None,
) -> Dict[str, Any]:
"""Full TikTok search: find videos, then fetch captions for top results.
Uses expand_tiktok_queries() to generate multiple search queries,
runs ScrapeCreators for each, and merges/deduplicates results by video ID.
Args:
topic: Search topic (raw topic, not planner's narrowed query)
from_date: Start date (YYYY-MM-DD)
to_date: End date (YYYY-MM-DD)
depth: 'quick', 'default', or 'deep'
token: ScrapeCreators API key
hashtags: Optional list of TikTok hashtags to search (without #)
creators: Optional list of TikTok creator handles to fetch videos from
Returns:
Dict with 'items' list. Each item has a 'caption_snippet' field.
"""
core_topic = _extract_core_subject(topic)
seen_ids: Set[str] = set()
items: List[Dict[str, Any]] = []
last_error = None
# Step 0a: Hashtag search (high-signal, runs first)
if hashtags and token:
for hashtag in hashtags:
raw_items = _hashtag_search(hashtag, token)
parsed = _parse_items(raw_items, core_topic)
for item in parsed:
vid = item.get("video_id", "")
if vid and vid not in seen_ids:
seen_ids.add(vid)
items.append(item)
# Step 0b: Creator profile videos (high-signal)
if creators and token:
for creator in creators:
raw_items = _profile_videos(creator, token)
parsed = _parse_items(raw_items, core_topic)
for item in parsed:
vid = item.get("video_id", "")
if vid and vid not in seen_ids:
seen_ids.add(vid)
items.append(item)
# Step 1: Multi-query keyword search — run ScrapeCreators for each expanded query
queries = expand_tiktok_queries(topic, depth)
for q in queries:
search_result = search_tiktok(q, from_date, to_date, depth, token)
if search_result.get("error"):
last_error = search_result["error"]
for item in search_result.get("items", []):
vid = item.get("video_id", "")
if vid and vid not in seen_ids:
seen_ids.add(vid)
items.append(item)
# Sort merged results by views descending
items.sort(key=lambda x: x.get("engagement", {}).get("views", 0), reverse=True)
if not items:
return {"items": [], "error": last_error}
# Step 2: Fetch captions for top N
captions = fetch_captions(items, token, depth)
# Step 3: Attach captions to items
for item in items:
vid = item["video_id"]
caption = captions.get(vid)
if caption:
item["caption_snippet"] = caption
return {"items": items, "error": last_error}
def parse_tiktok_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse TikTok search response to normalized format.
Returns:
List of item dicts ready for normalization.
"""
return response.get("items", [])
def _tiktok_total_engagement(item: Dict[str, Any]) -> int:
"""Total engagement for ranking which posts deserve comment enrichment."""
eng = item.get("engagement", {})
return (eng.get("views", 0) or 0) + (eng.get("likes", 0) or 0) + (eng.get("comments", 0) or 0)
def enrich_with_comments(
items: List[Dict[str, Any]],
token: str,
max_posts: int = 3,
max_comments: int = 5,
) -> List[Dict[str, Any]]:
"""Enrich top TikTok posts with comment data from ScrapeCreators.
For the top N posts by engagement, fetches comments via the SC API
and attaches them as a ``top_comments`` field on each item. Mirrors
youtube_yt.enrich_with_comments.
Args:
items: TikTok items from search_tiktok()
token: ScrapeCreators API key
max_posts: How many posts to enrich with comments
max_comments: Max comments to keep per post
Returns:
Items list (mutated in place) with top_comments added to enriched items.
"""
if not items or not token or max_posts <= 0:
return items
ranked = sorted(items, key=_tiktok_total_engagement, reverse=True)
top_items = ranked[:max_posts]
_log(f"Enriching comments for {len(top_items)} TikTok posts")
from concurrent.futures import ThreadPoolExecutor, as_completed
def _enrich_one(item: dict) -> bool:
post_url = item.get("url", "")
if not post_url:
return False
try:
comments = _fetch_post_comments(post_url, token, max_comments)
if comments:
item["top_comments"] = comments
return True
except Exception as exc:
_log(f"Comment enrichment failed for {post_url}: {exc}")
return False
enriched_count = 0
with ThreadPoolExecutor(max_workers=min(4, len(top_items))) as executor:
futures = {executor.submit(_enrich_one, item): item for item in top_items}
for future in as_completed(futures):
if future.result():
enriched_count += 1
_log(f"Enriched {enriched_count}/{len(top_items)} posts with comments")
return items
def _fetch_post_comments(
post_url: str,
token: str,
max_comments: int = 5,
) -> List[Dict[str, Any]]:
"""Fetch comments for a single TikTok post via ScrapeCreators.
SC endpoint: GET /v1/tiktok/video/comments?url=<video_url>
Response shape: { comments: [{text, user.nickname, digg_count, create_time, ...}], cursor, total }
Args:
post_url: Canonical TikTok post URL (share_url form works)
token: ScrapeCreators API key
max_comments: Maximum comments to return
Returns:
List of comment dicts with author, text, digg_count (likes), date.
Empty list on any error — comment failures never crash the pipeline.
"""
if not _requests:
try:
from urllib.parse import urlencode
params = urlencode({"url": post_url, "trim": "true"})
url = f"{SCRAPECREATORS_BASE}/video/comments?{params}"
headers = http.scrapecreators_headers(token)
headers["User-Agent"] = http.USER_AGENT
data = http.get(url, headers=headers, timeout=30, retries=2)
except Exception as exc:
_log(f"Comment fetch error (urllib) for {post_url}: {exc}")
return []
else:
try:
resp = _requests.get(
f"{SCRAPECREATORS_BASE}/video/comments",
params={"url": post_url, "trim": "true"},
headers=http.scrapecreators_headers(token),
timeout=30,
)
resp.raise_for_status()
data = resp.json()
except Exception as exc:
_log(f"Comment fetch error for {post_url}: {exc}")
return []
raw_comments = data.get("comments") or data.get("data") or []
# Sort by digg_count desc so normalize sees the highest-signal first.
raw_comments = sorted(
raw_comments,
key=lambda c: c.get("digg_count", 0) or 0,
reverse=True,
)
out: List[Dict[str, Any]] = []
for c in raw_comments[:max_comments]:
text = c.get("text") or ""
if not text:
continue
user = c.get("user") if isinstance(c.get("user"), dict) else {}
# Prefer unique_id (the @handle) over nickname (display name) so
# downstream render can cite @handle consistently across platforms.
author = user.get("unique_id") or user.get("nickname") or ""
create_time = c.get("create_time")
date_str = ""
if create_time:
try:
date_str = dates.timestamp_to_date(int(create_time)) or ""
except (ValueError, TypeError):
date_str = ""
out.append({
"author": author,
"text": text[:400],
"digg_count": c.get("digg_count", 0) or 0,
"date": date_str,
})
return out
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| add-product-e2e-evalpaperclipai/paperclipAdd or extend a Paperclip full-stack runner E2E workflow, fixture, matcher, or report evidence path for local or Daytona execution. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |
| add-runner-evalpaperclipai/paperclipAdd or extend a Paperclip Runner protocol evaluation definition, roster, assertion, or report fixture with provenance and narrow validation. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |
| diagnose-why-work-stoppedpaperclipai/paperclipCompatibility entry point for failed, stalled, looping, or over-recovered Paperclip work. Use fix-failed-paperclip-run for the full diagnosis, regression, repair, real verification, and green-PR workflow; honor an explicitly requested diagnosis-only scope. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |
| fix-failed-paperclip-runpaperclipai/paperclipDiagnose and fix failed, stuck, slow, or incorrectly blocked Paperclip tasks and runs. Use with a task/run URL or failure report to trace the root cause, prove a regression red then green, check second-order effects, verify the real workflow, and deliver a PR with passing checks. | Stars: 115k | Updated | Claude CodeCodex | Verified | Install |