meswork/yl
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add all dashboard projects

Browse Source
This commit is contained in:
meswork764
2026-05-23 14:07:11 +08:00
commit ba82fbfc14
3550 changed files with 467904 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

274
MacroinfManage_MES/tools/llm_api.py Normal file
View File

@@ -0,0 +1,274 @@
#!/usr/bin/env python3
import google.generativeai as genai
from openai import OpenAI, AzureOpenAI
from anthropic import Anthropic
import argparse
import os
from dotenv import load_dotenv
from pathlib import Path
import sys
import base64
from typing import Optional, Union, List
import mimetypes
def load_environment():
"""Load environment variables from .env files in order of precedence"""
# Order of precedence:
# 1. System environment variables (already loaded)
# 2. .env.local (user-specific overrides)
# 3. .env (project defaults)
# 4. .env.example (example configuration)
env_files = ['.env.local', '.env', '.env.example']
env_loaded = False
print("Current working directory:", Path('.').absolute(), file=sys.stderr)
print("Looking for environment files:", env_files, file=sys.stderr)
for env_file in env_files:
env_path = Path('.') / env_file
print(f"Checking {env_path.absolute()}", file=sys.stderr)
if env_path.exists():
print(f"Found {env_file}, loading variables...", file=sys.stderr)
load_dotenv(dotenv_path=env_path)
env_loaded = True
print(f"Loaded environment variables from {env_file}", file=sys.stderr)
# Print loaded keys (but not values for security)
with open(env_path) as f:
keys = [line.split('=')[0].strip() for line in f if '=' in line and not line.startswith('#')]
print(f"Keys loaded from {env_file}: {keys}", file=sys.stderr)
if not env_loaded:
print("Warning: No .env files found. Using system environment variables only.", file=sys.stderr)
print("Available system environment variables:", list(os.environ.keys()), file=sys.stderr)
# Load environment variables at module import
load_environment()
def encode_image_file(image_path: str) -> tuple[str, str]:
"""
Encode an image file to base64 and determine its MIME type.
Args:
image_path (str): Path to the image file
Returns:
tuple: (base64_encoded_string, mime_type)
"""
mime_type, _ = mimetypes.guess_type(image_path)
if not mime_type:
mime_type = 'image/png' # Default to PNG if type cannot be determined
with open(image_path, "rb") as image_file:
encoded_string = base64.b64encode(image_file.read()).decode('utf-8')
return encoded_string, mime_type
def create_llm_client(provider="openai"):
if provider == "openai":
api_key = os.getenv('OPENAI_API_KEY')
base_url = os.getenv('OPENAI_BASE_URL', "https://api.openai.com/v1")
if not api_key:
raise ValueError("OPENAI_API_KEY not found in environment variables")
return OpenAI(
api_key=api_key,
base_url=base_url
)
elif provider == "azure":
api_key = os.getenv('AZURE_OPENAI_API_KEY')
if not api_key:
raise ValueError("AZURE_OPENAI_API_KEY not found in environment variables")
return AzureOpenAI(
api_key=api_key,
api_version="2024-08-01-preview",
azure_endpoint="https://msopenai.openai.azure.com"
)
elif provider == "deepseek":
api_key = os.getenv('DEEPSEEK_API_KEY')
if not api_key:
raise ValueError("DEEPSEEK_API_KEY not found in environment variables")
return OpenAI(
api_key=api_key,
base_url="https://api.deepseek.com/v1",
)
elif provider == "siliconflow":
api_key = os.getenv('SILICONFLOW_API_KEY')
if not api_key:
raise ValueError("SILICONFLOW_API_KEY not found in environment variables")
return OpenAI(
api_key=api_key,
base_url="https://api.siliconflow.cn/v1"
)
elif provider == "anthropic":
api_key = os.getenv('ANTHROPIC_API_KEY')
if not api_key:
raise ValueError("ANTHROPIC_API_KEY not found in environment variables")
return Anthropic(
api_key=api_key
)
elif provider == "gemini":
api_key = os.getenv('GOOGLE_API_KEY')
if not api_key:
raise ValueError("GOOGLE_API_KEY not found in environment variables")
genai.configure(api_key=api_key)
return genai
elif provider == "local":
return OpenAI(
base_url="http://192.168.180.137:8006/v1",
api_key="not-needed"
)
else:
raise ValueError(f"Unsupported provider: {provider}")
def query_llm(prompt: str, client=None, model=None, provider="openai", image_path: Optional[str] = None) -> Optional[str]:
"""
Query an LLM with a prompt and optional image attachment.
Args:
prompt (str): The text prompt to send
client: The LLM client instance
model (str, optional): The model to use
provider (str): The API provider to use
image_path (str, optional): Path to an image file to attach
Returns:
Optional[str]: The LLM's response or None if there was an error
"""
if client is None:
client = create_llm_client(provider)
try:
# Set default model
if model is None:
if provider == "openai":
model = os.getenv('OPENAI_MODEL_DEPLOYMENT', 'gpt-4o')
elif provider == "azure":
model = os.getenv('AZURE_OPENAI_MODEL_DEPLOYMENT', 'gpt-4o-ms') # Get from env with fallback
elif provider == "deepseek":
model = "deepseek-chat"
elif provider == "siliconflow":
model = "deepseek-ai/DeepSeek-R1"
elif provider == "anthropic":
model = "claude-3-7-sonnet-20250219"
elif provider == "gemini":
model = "gemini-2.0-flash-exp"
elif provider == "local":
model = "Qwen/Qwen2.5-32B-Instruct-AWQ"
if provider in ["openai", "local", "deepseek", "azure", "siliconflow"]:
messages = [{"role": "user", "content": []}]
# Add text content
messages[0]["content"].append({
"type": "text",
"text": prompt
})
# Add image content if provided
if image_path:
if provider == "openai":
encoded_image, mime_type = encode_image_file(image_path)
messages[0]["content"] = [
{"type": "text", "text": prompt},
{"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{encoded_image}"}}
]
kwargs = {
"model": model,
"messages": messages,
"temperature": 0.7,
}
# Add o1-specific parameters
if model == "o1":
kwargs["response_format"] = {"type": "text"}
kwargs["reasoning_effort"] = "low"
del kwargs["temperature"]
response = client.chat.completions.create(**kwargs)
return response.choices[0].message.content
elif provider == "anthropic":
messages = [{"role": "user", "content": []}]
# Add text content
messages[0]["content"].append({
"type": "text",
"text": prompt
})
# Add image content if provided
if image_path:
encoded_image, mime_type = encode_image_file(image_path)
messages[0]["content"].append({
"type": "image",
"source": {
"type": "base64",
"media_type": mime_type,
"data": encoded_image
}
})
response = client.messages.create(
model=model,
max_tokens=1000,
messages=messages
)
return response.content[0].text
elif provider == "gemini":
model = client.GenerativeModel(model)
if image_path:
file = genai.upload_file(image_path, mime_type="image/png")
chat_session = model.start_chat(
history=[{
"role": "user",
"parts": [file, prompt]
}]
)
else:
chat_session = model.start_chat(
history=[{
"role": "user",
"parts": [prompt]
}]
)
response = chat_session.send_message(prompt)
return response.text
except Exception as e:
print(f"Error querying LLM: {e}", file=sys.stderr)
return None
def main():
parser = argparse.ArgumentParser(description='Query an LLM with a prompt')
parser.add_argument('--prompt', type=str, help='The prompt to send to the LLM', required=True)
parser.add_argument('--provider', choices=['openai','anthropic','gemini','local','deepseek','azure','siliconflow'], default='openai', help='The API provider to use')
parser.add_argument('--model', type=str, help='The model to use (default depends on provider)')
parser.add_argument('--image', type=str, help='Path to an image file to attach to the prompt')
args = parser.parse_args()
if not args.model:
if args.provider == 'openai':
args.model = "gpt-4o"
elif args.provider == "deepseek":
args.model = "deepseek-chat"
elif args.provider == "siliconflow":
args.model = "deepseek-ai/DeepSeek-R1"
elif args.provider == 'anthropic':
args.model = "claude-3-7-sonnet-20250219"
elif args.provider == 'gemini':
args.model = "gemini-2.0-flash-exp"
elif args.provider == 'azure':
args.model = os.getenv('AZURE_OPENAI_MODEL_DEPLOYMENT', 'gpt-4o-ms') # Get from env with fallback
client = create_llm_client(args.provider)
response = query_llm(args.prompt, client, model=args.model, provider=args.provider, image_path=args.image)
if response:
print(response)
else:
print("Failed to get response from LLM")
if __name__ == "__main__":
main()

56
MacroinfManage_MES/tools/screenshot_utils.py Normal file
View File

@@ -0,0 +1,56 @@
#!/usr/bin/env python3
import asyncio
from playwright.async_api import async_playwright
import os
import tempfile
from pathlib import Path
async def take_screenshot(url: str, output_path: str = None, width: int = 1280, height: int = 720) -> str:
"""
Take a screenshot of a webpage using Playwright.
Args:
url (str): The URL to take a screenshot of
output_path (str, optional): Path to save the screenshot. If None, saves to a temporary file.
width (int, optional): Viewport width. Defaults to 1280.
height (int, optional): Viewport height. Defaults to 720.
Returns:
str: Path to the saved screenshot
"""
if output_path is None:
# Create a temporary file with .png extension
temp_file = tempfile.NamedTemporaryFile(suffix='.png', delete=False)
output_path = temp_file.name
temp_file.close()
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
page = await browser.new_page(viewport={'width': width, 'height': height})
try:
await page.goto(url, wait_until='networkidle')
await page.screenshot(path=output_path, full_page=True)
finally:
await browser.close()
return output_path
def take_screenshot_sync(url: str, output_path: str = None, width: int = 1280, height: int = 720) -> str:
"""
Synchronous wrapper for take_screenshot.
"""
return asyncio.run(take_screenshot(url, output_path, width, height))
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description='Take a screenshot of a webpage')
parser.add_argument('url', help='URL to take screenshot of')
parser.add_argument('--output', '-o', help='Output path for screenshot')
parser.add_argument('--width', '-w', type=int, default=1280, help='Viewport width')
parser.add_argument('--height', '-H', type=int, default=720, help='Viewport height')
args = parser.parse_args()
output_path = take_screenshot_sync(args.url, args.output, args.width, args.height)
print(f"Screenshot saved to: {output_path}")

79
MacroinfManage_MES/tools/search_engine.py Normal file
View File

@@ -0,0 +1,79 @@
#!/usr/bin/env python3
import argparse
import sys
import time
from duckduckgo_search import DDGS
def search_with_retry(query, max_results=10, max_retries=3):
"""
Search using DuckDuckGo and return results with URLs and text snippets.
Args:
query (str): Search query
max_results (int): Maximum number of results to return
max_retries (int): Maximum number of retry attempts
"""
for attempt in range(max_retries):
try:
print(f"DEBUG: Searching for query: {query} (attempt {attempt + 1}/{max_retries})",
file=sys.stderr)
with DDGS() as ddgs:
results = list(ddgs.text(query, max_results=max_results))
if not results:
print("DEBUG: No results found", file=sys.stderr)
return []
print(f"DEBUG: Found {len(results)} results", file=sys.stderr)
return results
except Exception as e:
print(f"ERROR: Attempt {attempt + 1}/{max_retries} failed: {str(e)}", file=sys.stderr)
if attempt < max_retries - 1: # If not the last attempt
print(f"DEBUG: Waiting 1 second before retry...", file=sys.stderr)
time.sleep(1) # Wait 1 second before retry
else:
print(f"ERROR: All {max_retries} attempts failed", file=sys.stderr)
raise
def format_results(results):
"""Format and print search results."""
for i, r in enumerate(results, 1):
print(f"\n=== Result {i} ===")
print(f"URL: {r.get('href', 'N/A')}")
print(f"Title: {r.get('title', 'N/A')}")
print(f"Snippet: {r.get('body', 'N/A')}")
def search(query, max_results=10, max_retries=3):
"""
Main search function that handles search with retry mechanism.
Args:
query (str): Search query
max_results (int): Maximum number of results to return
max_retries (int): Maximum number of retry attempts
"""
try:
results = search_with_retry(query, max_results, max_retries)
if results:
format_results(results)
except Exception as e:
print(f"ERROR: Search failed: {str(e)}", file=sys.stderr)
sys.exit(1)
def main():
parser = argparse.ArgumentParser(description="Search using DuckDuckGo API")
parser.add_argument("query", help="Search query")
parser.add_argument("--max-results", type=int, default=10,
help="Maximum number of results (default: 10)")
parser.add_argument("--max-retries", type=int, default=3,
help="Maximum number of retry attempts (default: 3)")
args = parser.parse_args()
search(args.query, args.max_results, args.max_retries)
if __name__ == "__main__":
main()

207
MacroinfManage_MES/tools/web_scraper.py Normal file
View File

@@ -0,0 +1,207 @@
#!/usr/bin/env python3
import asyncio
import argparse
import sys
import os
from typing import List, Optional
from playwright.async_api import async_playwright
import html5lib
from multiprocessing import Pool
import time
from urllib.parse import urlparse
import logging
# Configure logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
stream=sys.stderr
)
logger = logging.getLogger(__name__)
async def fetch_page(url: str, context) -> Optional[str]:
"""Asynchronously fetch a webpage's content."""
page = await context.new_page()
try:
logger.info(f"Fetching {url}")
await page.goto(url)
await page.wait_for_load_state('networkidle')
content = await page.content()
logger.info(f"Successfully fetched {url}")
return content
except Exception as e:
logger.error(f"Error fetching {url}: {str(e)}")
return None
finally:
await page.close()
def parse_html(html_content: Optional[str]) -> str:
"""Parse HTML content and extract text with hyperlinks in markdown format."""
if not html_content:
return ""
try:
document = html5lib.parse(html_content)
result = []
seen_texts = set() # To avoid duplicates
def should_skip_element(elem) -> bool:
"""Check if the element should be skipped."""
# Skip script and style tags
if elem.tag in ['{http://www.w3.org/1999/xhtml}script',
'{http://www.w3.org/1999/xhtml}style']:
return True
# Skip empty elements or elements with only whitespace
if not any(text.strip() for text in elem.itertext()):
return True
return False
def process_element(elem, depth=0):
"""Process an element and its children recursively."""
if should_skip_element(elem):
return
# Handle text content
if hasattr(elem, 'text') and elem.text:
text = elem.text.strip()
if text and text not in seen_texts:
# Check if this is an anchor tag
if elem.tag == '{http://www.w3.org/1999/xhtml}a':
href = None
for attr, value in elem.items():
if attr.endswith('href'):
href = value
break
if href and not href.startswith(('#', 'javascript:')):
# Format as markdown link
link_text = f"[{text}]({href})"
result.append(" " * depth + link_text)
seen_texts.add(text)
else:
result.append(" " * depth + text)
seen_texts.add(text)
# Process children
for child in elem:
process_element(child, depth + 1)
# Handle tail text
if hasattr(elem, 'tail') and elem.tail:
tail = elem.tail.strip()
if tail and tail not in seen_texts:
result.append(" " * depth + tail)
seen_texts.add(tail)
# Start processing from the body tag
body = document.find('.//{http://www.w3.org/1999/xhtml}body')
if body is not None:
process_element(body)
else:
# Fallback to processing the entire document
process_element(document)
# Filter out common unwanted patterns
filtered_result = []
for line in result:
# Skip lines that are likely to be noise
if any(pattern in line.lower() for pattern in [
'var ',
'function()',
'.js',
'.css',
'google-analytics',
'disqus',
'{',
'}'
]):
continue
filtered_result.append(line)
return '\n'.join(filtered_result)
except Exception as e:
logger.error(f"Error parsing HTML: {str(e)}")
return ""
async def process_urls(urls: List[str], max_concurrent: int = 5) -> List[str]:
"""Process multiple URLs concurrently."""
async with async_playwright() as p:
browser = await p.chromium.launch()
try:
# Create browser contexts
n_contexts = min(len(urls), max_concurrent)
contexts = [await browser.new_context() for _ in range(n_contexts)]
# Create tasks for each URL
tasks = []
for i, url in enumerate(urls):
context = contexts[i % len(contexts)]
task = fetch_page(url, context)
tasks.append(task)
# Gather results
html_contents = await asyncio.gather(*tasks)
# Parse HTML contents in parallel
with Pool() as pool:
results = pool.map(parse_html, html_contents)
return results
finally:
# Cleanup
for context in contexts:
await context.close()
await browser.close()
def validate_url(url: str) -> bool:
"""Validate if the given string is a valid URL."""
try:
result = urlparse(url)
return all([result.scheme, result.netloc])
except:
return False
def main():
parser = argparse.ArgumentParser(description='Fetch and extract text content from webpages.')
parser.add_argument('urls', nargs='+', help='URLs to process')
parser.add_argument('--max-concurrent', type=int, default=5,
help='Maximum number of concurrent browser instances (default: 5)')
parser.add_argument('--debug', action='store_true',
help='Enable debug logging')
args = parser.parse_args()
if args.debug:
logger.setLevel(logging.DEBUG)
# Validate URLs
valid_urls = []
for url in args.urls:
if validate_url(url):
valid_urls.append(url)
else:
logger.error(f"Invalid URL: {url}")
if not valid_urls:
logger.error("No valid URLs provided")
sys.exit(1)
start_time = time.time()
try:
results = asyncio.run(process_urls(valid_urls, args.max_concurrent))
# Print results to stdout
for url, text in zip(valid_urls, results):
print(f"\n=== Content from {url} ===")
print(text)
print("=" * 80)
logger.info(f"Total processing time: {time.time() - start_time:.2f}s")
except Exception as e:
logger.error(f"Error during execution: {str(e)}")
sys.exit(1)
if __name__ == '__main__':
main()