One post tagged with "Reliability"

# Accuracy = matching the exchange's official data
exchange_price = 43251.50  # Direct from Binance API
parser_price = 43251.50    # From your data source

accuracy = 100% if exchange_price == parser_price else 0%

# ❌ BAD: Scraping HTML (30-60s delay)
import requests
from bs4 import BeautifulSoup

html = requests.get("https://exchange.com/markets/BTC-USD").text
soup = BeautifulSoup(html, 'html.parser')
price = float(soup.find("div", class_="price").text)

# Issues:
# - Price from 30-60 seconds ago
# - HTML may be cached by CDN
# - No timestamp information
# - Can't verify freshness

# ❌ BAD: Fragile HTML parsing
price_text = soup.find("span", class_="price-value").text
# "$ 43,251.50 USD"

# Naive parsing
price = float(price_text.replace("$", "").replace(",", ""))
# Works... until exchange changes format to "43.251,50" (EU format)
# Result: Crash or wrong data

# ✅ GOOD: Direct API access with validation
from stockapi import BinanceParser

parser = BinanceParser()
ticker = parser.get_ticker("BTCUSDT")

print(f"Price: ${ticker['price']}")
print(f"Timestamp: {ticker['timestamp']}")  # Server timestamp
print(f"Data age: {ticker['age_ms']}ms")    # Calculated freshness

# Guarantees:
# - Direct from exchange API
# - Validated against schema
# - Timestamp included
# - Sub-second freshness
# - 99.99% accuracy rate

# Compare provider data against exchange API
import time
from stockapi import BinanceParser

parser = BinanceParser()
correct = 0
total = 0

for _ in range(1000):
    # Get from both sources simultaneously
    parser_data = parser.get_ticker("BTCUSDT")
    exchange_data = requests.get(
        "https://api.binance.com/api/v3/ticker/price?symbol=BTCUSDT"
    ).json()

    if parser_data['price'] == float(exchange_data['price']):
        correct += 1
    total += 1

    time.sleep(1)

accuracy = (correct / total) * 100
print(f"Accuracy: {accuracy}%")

# StockAPI Results:
# - Binance: 99.99% (5 mismatches in 50,000 samples)
# - Coinbase: 99.98%
# - NYSE: 99.95%
#
# Typical DIY Scraping:
# - 85-95% accuracy (frequent parsing errors)

# Latency = time from event to data availability
event_time = 1699564723.145  # Exchange timestamp
receive_time = 1699564723.198  # Local timestamp

latency_ms = (receive_time - event_time) * 1000
# Target: <100ms for real-time trading

# Arbitrage opportunity window
binance_price = 43250.00  # Updated at T+0ms
coinbase_price = 43270.00 # Updated at T+50ms (50ms latency)

spread = 43270 - 43250 = $20 profit opportunity

# But...
# High-frequency traders with 10ms latency already took it
# You arrive at T+50ms: opportunity gone
# Result: Missed trade

# ✅ Real-world latency measurement
from stockapi import BinanceParser
import time

parser = BinanceParser()
latencies = []

for update in parser.stream_ticker("BTCUSDT"):
    exchange_time = update['timestamp']
    local_time = time.time() * 1000

    latency = local_time - exchange_time
    latencies.append(latency)

    if len(latencies) == 1000:
        break

# Calculate percentiles
p50 = sorted(latencies)[500]   # Median
p95 = sorted(latencies)[950]   # 95th percentile
p99 = sorted(latencies)[990]   # 99th percentile

print(f"Median latency: {p50:.2f}ms")
print(f"P95 latency: {p95:.2f}ms")
print(f"P99 latency: {p99:.2f}ms")

# StockAPI Results (WebSocket):
# - P50: 35ms
# - P95: 85ms
# - P99: 150ms
#
# DIY Scraping (REST polling):
# - P50: 650ms
# - P95: 2400ms
# - P99: 5000ms+

# Uptime = percentage of time data is available
uptime_percentage = (operational_time / total_time) * 100

# ❌ BAD: Crashes on first error
import requests

while True:
    response = requests.get("https://api.binance.com/ticker")
    data = response.json()
    # Process data...

# What happens when:
# - Network hiccup: CRASH
# - API rate limit: CRASH
# - Server timeout: CRASH
# - Invalid JSON: CRASH
#
# Requires manual restart
# 95% uptime at best

# ❌ BAD: Fails silently, returns stale data
cached_price = 43250.00

try:
    response = requests.get("https://api.binance.com/ticker", timeout=1)
    price = response.json()['price']
except:
    price = cached_price  # Return old data!

# Problems:
# - Trading on stale data
# - No error notification
# - Silent degradation
# - False confidence

# ✅ GOOD: Automatic recovery with monitoring
from stockapi import BinanceParser

parser = BinanceParser(
    retry_attempts=5,
    retry_delay=1.0,
    circuit_breaker=True,  # Stop on repeated failures
    health_check_interval=60,
)

# Real-time health monitoring
if parser.is_healthy():
    ticker = parser.get_ticker("BTCUSDT")
else:
    # Parser detected issues and switched to backup
    send_alert("Primary parser unhealthy, using backup")

# Handles automatically:
# - Network failures
# - API rate limits
# - Server timeouts
# - Invalid responses
# - 99.9% uptime guaranteed

# 30-day uptime tracking
import time
from stockapi import BinanceParser

parser = BinanceParser()
successful_calls = 0
failed_calls = 0

# Check every minute for 30 days
for _ in range(43200):  # 30 days * 24 hours * 60 minutes
    try:
        ticker = parser.get_ticker("BTCUSDT", timeout=5)
        if ticker and ticker['price'] > 0:
            successful_calls += 1
        else:
            failed_calls += 1
    except:
        failed_calls += 1

    time.sleep(60)

uptime = (successful_calls / (successful_calls + failed_calls)) * 100
print(f"30-day uptime: {uptime}%")

# StockAPI Results:
# - 99.92% uptime (35 minutes downtime/month)
#
# DIY Scraping Results:
# - 85-95% uptime (36-108 hours downtime/month)

# Completeness = percentage of expected data fields present
expected_fields = [
    'symbol', 'price', 'volume', 'high', 'low',
    'open', 'close', 'timestamp', 'change_24h'
]

received_fields = list(data.keys())
completeness = (
    len(set(expected_fields) & set(received_fields)) /
    len(expected_fields)
) * 100

# ❌ BAD: Scraping misses fields
html_data = {
    'price': 43250.00,
    'symbol': 'BTCUSDT',
    # Missing: volume, timestamp, high/low, change
}

# Can't calculate:
# - Price momentum (no change %)
# - Volume trend (no volume)
# - Data freshness (no timestamp)
# - Daily range (no high/low)

# ✅ GOOD: Complete data set
from stockapi import BinanceParser

parser = BinanceParser()
ticker = parser.get_ticker("BTCUSDT")

print(ticker)
# {
#   'symbol': 'BTCUSDT',
#   'price': 43250.00,
#   'volume_24h': 28450.5,
#   'high_24h': 44100.00,
#   'low_24h': 42800.00,
#   'open_24h': 43000.00,
#   'close_24h': 43250.00,
#   'change_24h': 0.58,
#   'change_percent_24h': '0.58%',
#   'timestamp': 1699564723145,
#   'bid': 43249.50,
#   'ask': 43250.50,
#   'spread': 1.00,
# }

# 100% completeness
# All fields guaranteed
# Validated schema

# When your scraper was down, can you recover the data?

# ❌ DIY Scraping: Data is lost forever
downtime_start = "2024-03-15 14:30:00"
downtime_end = "2024-03-15 14:42:00"
# 12 minutes of missing data
# Can't recover: exchange APIs don't provide historical tick data
# Result: Gaps in your database

# ✅ StockAPI: Automatic backfill
parser = BinanceParser()
historical_data = parser.get_ticker_history(
    symbol="BTCUSDT",
    start_time="2024-03-15 14:30:00",
    end_time="2024-03-15 14:42:00",
    interval="1m"
)
# Complete data recovered
# No gaps in historical analysis