
import kagglehub

# Download latest version
path = kagglehub.dataset_download("franoisgeorgesjulien/crypto")

print("Path to dataset files:", path)

import os

dataset_path = "/root/.cache/kagglehub/datasets/franoisgeorgesjulien/crypto/versions/3"
files = os.listdir(dataset_path)
print(files)

import pandas as pd

btc = pd.read_csv(os.path.join(dataset_path, 'Binance_BTCUSDT_1h (1).csv'))
eth = pd.read_csv(os.path.join(dataset_path, 'Binance_ETHUSDT_1h (1).csv'))
doge = pd.read_csv(os.path.join(dataset_path, 'Binance_DOGEUSDT_1h (1).csv'))

print(btc.head())
print(eth.head())
print(doge.head())

btc.head()

eth.head()

doge.head()

from google.colab import files

uploaded = files.upload()  # This will prompt file upload

"""**Twitter_Sentiments Pre-Processing to extraxt Sentiment Scores**"""

import pandas as pd

file_path = "crypto-query-tweets1(in).csv"
Twitter_sentiment_df = pd.read_csv(file_path)

# converting with the expected format, handling errors
Twitter_sentiment_df['Date'] = pd.to_datetime(Twitter_sentiment_df['Date'], format="%d/%m/%Y %H:%M", errors='coerce')

# Checked for rows where the conversion failed
print("Rows with conversion issues:")
print(Twitter_sentiment_df[Twitter_sentiment_df['Date'].isna()])

# Now format the column properly
Twitter_sentiment_df['Date'] = Twitter_sentiment_df['Date'].dt.strftime("%Y-%m-%d %H:%M:%S")

# Display the first few rows to check
print(Twitter_sentiment_df.head())

import pandas as pd
import numpy as np
import re
from textblob import TextBlob
from sklearn.preprocessing import MinMaxScaler

# Load dataset (assuming it's already a Pandas DataFrame)
df = Twitter_sentiment_df.copy()


#  Drop Unnecessary Columns
df.drop(['username', 'user_location', 'user_description'], axis=1, inplace=True)

#  Handle Missing Values
df.fillna({'followers_count': 0, 'following_count': 0, 'tweet_like_count': 0,
           'tweet_retweet_count': 0, 'tweet_reply_count': 0, 'tweet_quote_count': 0}, inplace=True)

#  Convert 'verified' Column to Binary
df['verified'] = df['verified'].astype(int)

# 4️ Preprocess Tweet Text (Cleaning + Sentiment Analysis)
def clean_text(text):
    text = re.sub(r'http\S+|www\S+', '', text)  # Remove URLs
    text = re.sub(r'@[A-Za-z0-9_]+', '', text)  # Remove mentions
    text = re.sub(r'#[A-Za-z0-9_]+', '', text)  # Remove hashtags
    text = re.sub(r'[^a-zA-Z0-9 ]', '', text)  # Remove special characters
    return text.lower().strip()

def get_sentiment(text):
    return TextBlob(text).sentiment.polarity  # Returns sentiment score (-1 to 1)

df['clean_tweet'] = df['tweet_text'].astype(str).apply(clean_text)
df['sentiment_score'] = df['clean_tweet'].apply(get_sentiment)

# Convert 'Date' Column to Datetime & Extract Features
df['Date'] = pd.to_datetime(df['Date'])
df['hour'] = df['Date'].dt.hour
df['day'] = df['Date'].dt.day
df['weekday'] = df['Date'].dt.weekday  # 0 = Monday, 6 = Sunday

# Convert 'Date' Column to Datetime & Extract Features
df['Date'] = pd.to_datetime(df['Date'])
df['hour'] = df['Date'].dt.hour
df['day'] = df['Date'].dt.day
df['weekday'] = df['Date'].dt.weekday  # 0 = Monday, 6 = Sunday


# Normalize Numeric Features
scaler = MinMaxScaler()
numeric_cols = ['followers_count', 'following_count', 'tweet_like_count',
                'tweet_retweet_count', 'tweet_reply_count', 'tweet_quote_count', 'sentiment_score']
df[numeric_cols] = scaler.fit_transform(df[numeric_cols])

# Show all columns in the dataset
pd.set_option('display.max_columns', None)  # Ensure all columns are visible
df.head(5)  # Display first few rows

"""**Checking the Columns and Shapes to Merge All the Datasets.**"""

print("BTC Columns:", btc.columns.tolist())
print("ETH Columns:", eth.columns.tolist())
print("DOGE Columns:", doge.columns.tolist())
print("Twitter Sentiment Columns:", df.columns.tolist())

print ("BTC Shape:", btc.shape)
print("ETH Shape:", eth.shape)
print("DOGE Shape:", doge.shape)
print("Twitter Sentiment Shape:", df.shape)

# Find the minimum shape length
min_length = min(len(btc), len(eth), len(doge), len(df))

# Truncate the datasets to the minimum length
btc = btc.iloc[:min_length]
eth = eth.iloc[:min_length]
doge = doge.iloc[:min_length]
df = df.iloc[:min_length]

# Check the shapes after truncation
print(btc.shape, eth.shape, doge.shape, df.shape)

print ("BTC Shape:", btc.shape)
print("ETH Shape:", eth.shape)
print("DOGE Shape:", doge.shape)
print("Twitter Sentiment Shape:", df.shape)

"""**Alligning Dataset based on TimeStamp and Merging it into ONE DATAFRAME**"""

btc['Date'] = pd.to_datetime(btc['Date'])
eth['Date'] = pd.to_datetime(eth['Date'])
doge['Date'] = pd.to_datetime(doge['Date'])
df['Date'] = pd.to_datetime(df['Date'])

# Merge the datasets on 'Date' column
merged_data = pd.merge(btc, eth, on='Date', suffixes=('_btc', '_eth'))
merged_data = pd.merge(merged_data, doge, on='Date', suffixes=('', '_doge'))
merged_data = pd.merge(merged_data, df, on='Date', suffixes=('', '_Twitter'))

import pandas as pd

# Set display options
pd.set_option('display.max_columns', None)  # Show all columns
pd.set_option('display.max_colwidth', 20)   # Limit maximum width of each column (e.g., 20 characters)
pd.set_option('display.width', 1000)         # Set display width

# Display first 5 rows
merged_data.head()

merged_data.head()

"""# **PRE-PROCESSING of Merged DATASET**"""

merged_data.describe()

merged_data.info()

merged_data.nunique()

merged_data.isnull().sum()

merged_data.shape

import pandas as pd

# Assuming 'merged_data' is your DataFrame and it has a 'Date' column
merged_data['Date'] = pd.to_datetime(merged_data['Date'])  # Ensuring the 'Date' column is in datetime format

# Getting the first and last date from the 'Date' column
first_date = merged_data['Date'].min()
last_date = merged_data['Date'].max()

print("First Date:", first_date)
print("Last Date:", last_date)

import matplotlib.pyplot as plt
import seaborn as sns

# List of columns to check for outliers (just the selected ones)
columns_to_check = ['Close_btc', 'Close_eth', 'Close', 'sentiment_score']

# Plotting boxplots for the selected columns
plt.figure(figsize=(10, 6))
for i, column in enumerate(columns_to_check, 1):
    plt.subplot(2, 2, i)  # Arrange plots in a 2x2 grid
    sns.boxplot(x=merged_data[column])
    plt.title(f'Boxplot of {column}')
plt.tight_layout()
plt.show()

# Function to remove outliers using the IQR method
def remove_outliers_iqr(df, columns):
    for column in columns:
        # Calculate Q1 (25th percentile) and Q3 (75th percentile)
        Q1 = df[column].quantile(0.25)
        Q3 = df[column].quantile(0.75)

        # Calculate the IQR
        IQR = Q3 - Q1

        # Define the outlier bounds
        lower_bound = Q1 - 1.5 * IQR
        upper_bound = Q3 + 1.5 * IQR

        # Remove outliers by filtering the data
        df = df[(df[column] >= lower_bound) & (df[column] <= upper_bound)]

    return df

# Apply the function to remove outliers
merged_data_cleaned = remove_outliers_iqr(merged_data, columns_to_check)

# Display the shape of the cleaned dataset
print(f"Original dataset shape: {merged_data.shape}")
print(f"Cleaned dataset shape: {merged_data_cleaned.shape}")

"""**Deleting Unnecessary Columns**"""

# List of columns to delete
columns_to_delete = [
    'tweet_text', 'followers_count', 'following_count',
    'tweet_like_count', 'tweet_retweet_count', 'tweet_reply_count',
    'tweet_quote_count', 'verified', 'clean_tweet', 'clean_tweet'
]

# Drop the columns from the DataFrame
merged_data_cleaned = merged_data.drop(columns=columns_to_delete)

# Display the cleaned dataset
print(f"Dataset after deleting unnecessary columns:")
print(merged_data_cleaned.head())

merged_data_cleaned.head()

merged_data_cleaned.describe()

merged_data_cleaned.info()

"""# **Exploratory Data Analysis (EDA)**"""

import matplotlib.pyplot as plt
import pandas as pd

# Convert 'Date' to datetime if not already done
merged_data_cleaned['Date'] = pd.to_datetime(merged_data_cleaned['Date'])

# Filter data for the last 3 years
end_date = merged_data_cleaned['Date'].max()
start_date = end_date - pd.DateOffset(years=3)

# Filter the dataset
recent_data = merged_data_cleaned[(merged_data_cleaned['Date'] >= start_date) & (merged_data_cleaned['Date'] <= end_date)]

# Resample the data by year (you can change this to 'M' for monthly if preferred)
recent_data_resampled = recent_data.resample('Y', on='Date').agg({
    'Open_btc': 'mean',
    'Close_btc': 'mean'
})

# Plotting the bar graph
recent_data_resampled.plot(kind='bar', figsize=(10, 6), width=0.8)
plt.title('BTC Open and Close Prices for the Last 3 Years', fontsize=14)
plt.xlabel('Year', fontsize=12)
plt.ylabel('Price (USD)', fontsize=12)
plt.xticks(rotation=45)
plt.tight_layout()

# Show the plot
plt.show()

import plotly.graph_objects as go

# Ensure 'Date' column is in datetime format
merged_data_cleaned['Date'] = pd.to_datetime(merged_data_cleaned['Date'])

# Filter data for the last 3 years
end_date = merged_data_cleaned['Date'].max()
start_date = end_date - pd.DateOffset(years=3)

# Filter the dataset for the last 3 years
recent_data = merged_data_cleaned[(merged_data_cleaned['Date'] >= start_date) & (merged_data_cleaned['Date'] <= end_date)]

# Extract month-wise data for each year
recent_data['Year'] = recent_data['Date'].dt.year
recent_data['Month'] = recent_data['Date'].dt.month

# Plot for each year separately
fig = go.Figure()

# Loop through the years and add bar traces
for year in recent_data['Year'].unique():
    # Filter data for each year
    year_data = recent_data[recent_data['Year'] == year]

    # Group by month and calculate the mean for Open and Close
    monthwise = year_data.groupby('Month').agg({
        'Open_btc': 'mean',
        'Close_btc': 'mean'
    }).reset_index()

    # Plot Open and Close for each year
    fig.add_trace(go.Bar(
        x=monthwise['Month'],
        y=monthwise['Open_btc'],
        name=f'{year} Open BTC',
        marker_color='crimson'
    ))
    fig.add_trace(go.Bar(
        x=monthwise['Month'],
        y=monthwise['Close_btc'],
        name=f'{year} Close BTC',
        marker_color='lightsalmon'
    ))

# Update the layout for the plot
fig.update_layout(
    barmode='group',
    xaxis_tickangle=-45,
    title='Monthwise Comparison between BTC Open and Close Price for the Last 3 Years',
    xaxis=dict(tickmode='array', tickvals=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], ticktext=['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']),
    xaxis_title='Month',
    yaxis_title='Price (USD)'
)

# Show the plot
fig.show()

import plotly.graph_objects as go
import pandas as pd

# Ensure 'Date' column is in datetime format
merged_data_cleaned['Date'] = pd.to_datetime(merged_data_cleaned['Date'])

# Filter data for the last 3 years
end_date = merged_data_cleaned['Date'].max()
start_date = end_date - pd.DateOffset(years=3)

# Filter the dataset for the last 3 years (ETH)
recent_data_eth = merged_data_cleaned[(merged_data_cleaned['Date'] >= start_date) & (merged_data_cleaned['Date'] <= end_date)]

# Extract month-wise data for each year
recent_data_eth['Year'] = recent_data_eth['Date'].dt.year
recent_data_eth['Month'] = recent_data_eth['Date'].dt.month

# Plot for each year separately (ETH)
fig_eth = go.Figure()

# Loop through the years and add bar traces for ETH
for year in recent_data_eth['Year'].unique():
    # Filter data for each year
    year_data_eth = recent_data_eth[recent_data_eth['Year'] == year]

    # Group by month and calculate the mean for Open and Close (ETH)
    monthwise_eth = year_data_eth.groupby('Month').agg({
        'Open_eth': 'mean',
        'Close_eth': 'mean'
    }).reset_index()

    # Plot Open and Close for ETH
    fig_eth.add_trace(go.Bar(
        x=monthwise_eth['Month'],
        y=monthwise_eth['Open_eth'],
        name=f'{year} Open ETH',
        marker_color='forestgreen'
    ))
    fig_eth.add_trace(go.Bar(
        x=monthwise_eth['Month'],
        y=monthwise_eth['Close_eth'],
        name=f'{year} Close ETH',
        marker_color='mediumseagreen'
    ))

# Update the layout for the plot
fig_eth.update_layout(
    barmode='group',
    xaxis_tickangle=-45,
    title='Monthwise Comparison between ETH Open and Close Prices for the Last 3 Years',
    xaxis=dict(tickmode='array', tickvals=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], ticktext=['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']),
    xaxis_title='Month',
    yaxis_title='Price (USD)'
)

# Show the plot
fig_eth.show()

import plotly.graph_objects as go
import pandas as pd

# Ensure 'Date' column is in datetime format
merged_data_cleaned['Date'] = pd.to_datetime(merged_data_cleaned['Date'])

# Filter data for the last 3 years
end_date = merged_data_cleaned['Date'].max()
start_date = end_date - pd.DateOffset(years=3)

# Filter the dataset for the last 3 years (DOGE)
recent_data_doge = merged_data_cleaned[(merged_data_cleaned['Date'] >= start_date) & (merged_data_cleaned['Date'] <= end_date)]

# Extract month-wise data for each year
recent_data_doge['Year'] = recent_data_doge['Date'].dt.year
recent_data_doge['Month'] = recent_data_doge['Date'].dt.month

# Plot for each year separately (DOGE)
fig_doge = go.Figure()

# Loop through the years and add bar traces for DOGE
for year in recent_data_doge['Year'].unique():
    # Filter data for each year
    year_data_doge = recent_data_doge[recent_data_doge['Year'] == year]

    # Group by month and calculate the mean for Open and Close (DOGE)
    monthwise_doge = year_data_doge.groupby('Month').agg({
        'Open': 'mean',
        'Close': 'mean'
    }).reset_index()

    # Plot Open and Close for DOGE
    fig_doge.add_trace(go.Bar(
        x=monthwise_doge['Month'],
        y=monthwise_doge['Open'],
        name=f'{year} Open DOGE',
        marker_color='gold'
    ))
    fig_doge.add_trace(go.Bar(
        x=monthwise_doge['Month'],
        y=monthwise_doge['Close'],
        name=f'{year} Close DOGE',
        marker_color='yellowgreen'
    ))

# Update the layout for the plot
fig_doge.update_layout(
    barmode='group',
    xaxis_tickangle=-45,
    title='Monthwise Comparison between DOGE Open and Close Prices for the Last 3 Years',
    xaxis=dict(tickmode='array', tickvals=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], ticktext=['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']),
    xaxis_title='Month',
    yaxis_title='Price (USD)'
)

# Show the plot
fig_doge.show()

import plotly.express as px
import pandas as pd
from itertools import cycle

# Ensure 'Date' column is in datetime format
merged_data_cleaned['Date'] = pd.to_datetime(merged_data_cleaned['Date'])

# Filter data for the recent year
end_date = merged_data_cleaned['Date'].max()
start_date = end_date - pd.DateOffset(years=1)

# Filter the dataset for the recent year (BTC)
recent_data_btc = merged_data_cleaned[(merged_data_cleaned['Date'] >= start_date) & (merged_data_cleaned['Date'] <= end_date)]

# Plot the BTC parameters: Open, Close, High, Low
names = cycle(['BTC Open Price', 'BTC Close Price', 'BTC High Price', 'BTC Low Price'])

# Create line plot for BTC parameters
fig = px.line(recent_data_btc, x='Date', y=['Open_btc', 'Close_btc', 'High_btc', 'Low_btc'],
              labels={'Date': 'Date', 'value': 'BTC Value'},
              title='BTC Parameter Analysis for the Recent Year')

# Update legend and trace names
fig.update_layout(font_size=15, font_color='black', legend_title_text='BTC Parameters')
fig.for_each_trace(lambda t: t.update(name=next(names)))

# Update axes
fig.update_xaxes(showgrid=False)
fig.update_yaxes(showgrid=False)

# Show the plot
fig.show()

import plotly.express as px
import pandas as pd
from itertools import cycle

# Ensure 'Date' column is in datetime format
merged_data_cleaned['Date'] = pd.to_datetime(merged_data_cleaned['Date'])

# Filter data for the recent year (ETH)
end_date = merged_data_cleaned['Date'].max()
start_date = end_date - pd.DateOffset(years=1)

# Filter the dataset for the recent year (ETH)
recent_data_eth = merged_data_cleaned[(merged_data_cleaned['Date'] >= start_date) & (merged_data_cleaned['Date'] <= end_date)]

# Plot the ETH parameters: Open, Close, High, Low
names = cycle(['ETH Open Price', 'ETH Close Price', 'ETH High Price', 'ETH Low Price'])

# Create line plot for ETH parameters
fig_eth = px.line(recent_data_eth, x='Date', y=['Open_eth', 'Close_eth', 'High_eth', 'Low_eth'],
                  labels={'Date': 'Date', 'value': 'ETH Value'},
                  title='ETH Parameter Analysis for the Recent Year')

# Update legend and trace names
fig_eth.update_layout(font_size=15, font_color='black', legend_title_text='ETH Parameters')
fig_eth.for_each_trace(lambda t: t.update(name=next(names)))

# Update axes
fig_eth.update_xaxes(showgrid=False)
fig_eth.update_yaxes(showgrid=False)

# Show the plot
fig_eth.show()

import plotly.express as px
import pandas as pd
from itertools import cycle

# Ensure 'Date' column is in datetime format
merged_data_cleaned['Date'] = pd.to_datetime(merged_data_cleaned['Date'])

# Filter data for the recent year (DOGE)
end_date = merged_data_cleaned['Date'].max()
start_date = end_date - pd.DateOffset(years=1)

# Filter the dataset for the recent year (DOGE)
recent_data_doge = merged_data_cleaned[(merged_data_cleaned['Date'] >= start_date) & (merged_data_cleaned['Date'] <= end_date)]

# Plot the DOGE parameters: Open, Close, High, Low
names = cycle(['DOGE Open Price', 'DOGE Close Price', 'DOGE High Price', 'DOGE Low Price'])

# Create line plot for DOGE parameters
fig_doge = px.line(recent_data_doge, x='Date', y=['Open', 'Close', 'High', 'Low'],
                   labels={'Date': 'Date', 'value': 'DOGE Value'},
                   title='DOGE Parameter Analysis for the Recent Year')

# Update legend and trace names
fig_doge.update_layout(font_size=15, font_color='black', legend_title_text='DOGE Parameters')
fig_doge.for_each_trace(lambda t: t.update(name=next(names)))

# Update axes
fig_doge.update_xaxes(showgrid=False)
fig_doge.update_yaxes(showgrid=False)

# Show the plot
fig_doge.show()

"""# BUILDING HYBRID MODELS
#1 - VAR - LSTM (Hybrid Model)
"""

merged_data_cleaned.head()

merged_data_cleaned.head()

print(merged_data_cleaned.columns)

merged_data_cleaned.head()

print(merged_data_cleaned.index.name)  # This will tell you if the index is named 'Date'

merged_data_cleaned.reset_index(inplace=True)

import pandas as pd
from sklearn.preprocessing import StandardScaler

# Sort the data by date
merged_data_cleaned.sort_values('Date', inplace=True)

# Set Date as the index
merged_data_cleaned.set_index('Date', inplace=True)

# Select closing prices only for VAR
crypto_data = pd.concat([
    merged_data_cleaned['Close_btc'],  # BTC Close
    merged_data_cleaned['Close_eth'],  # ETH Close
    merged_data_cleaned['Close'],     # DOGE Close (assuming 'Close' column is for DOGE)
], axis=1)

# Rename the columns for clarity
crypto_data.columns = ['BTC_Close', 'ETH_Close', 'DOGE_Close']

# Check for any missing values and forward-fill if necessary
crypto_data.ffill(inplace=True)

# Optional: Use sentiment as an exogenous variable
sentiment_data = pd.concat([
    merged_data_cleaned['sentiment_score'],  # Sentiment score (can be used for any crypto)
], axis=1)

sentiment_data.columns = ['Sentiment']
sentiment_data.ffill(inplace=True)

# Scale the data using StandardScaler
scaler = StandardScaler()

# Scale the closing prices and sentiment scores
scaled_crypto_data = scaler.fit_transform(crypto_data)
scaled_sentiment_data = scaler.fit_transform(sentiment_data)

# Convert scaled data back to DataFrame for easier handling
scaled_crypto_data = pd.DataFrame(scaled_crypto_data, columns=crypto_data.columns, index=crypto_data.index)
scaled_sentiment_data = pd.DataFrame(scaled_sentiment_data, columns=sentiment_data.columns, index=sentiment_data.index)

# Display sample of prepared and scaled data
print(scaled_crypto_data.head())
print(scaled_sentiment_data.head())

from statsmodels.tsa.api import VAR
import matplotlib.pyplot as plt
import pandas as pd

# Step 1: Prepare Data for VAR (BTC, ETH, DOGE Close Prices)
crypto_data = pd.concat([
    merged_data_cleaned['Close_btc'],  # BTC Close
    merged_data_cleaned['Close_eth'],  # ETH Close
    merged_data_cleaned['Close'],     # DOGE Close (assuming 'Close' column is for DOGE)
], axis=1)

crypto_data.columns = ['BTC_Close', 'ETH_Close', 'DOGE_Close']

# Check for missing values and forward fill if necessary
crypto_data.ffill(inplace=True)

# Step 2: Determine Optimal Lag for the VAR Model
model = VAR(crypto_data)
lag_order_results = model.select_order(maxlags=15)
print(lag_order_results.summary())

# Choose optimal lag based on AIC/BIC (you can choose based on other criteria like BIC if needed)
optimal_lag = lag_order_results.aic
print(f"Optimal Lag (AIC): {optimal_lag}")

# Step 3: Fit the VAR Model
var_model = model.fit(optimal_lag)
print(var_model.summary())

# Step 4: Forecasting using VAR (next 24 steps, adjust as needed)
forecast_steps = 24  # Example: Forecast next 24 time steps (hours)
forecast_input = crypto_data.values[-optimal_lag:]  # Use the last `optimal_lag` number of observations as input
forecast = var_model.forecast(y=forecast_input, steps=forecast_steps)

# Convert forecast to DataFrame
forecast_df = pd.DataFrame(forecast, columns=['BTC_Forecast', 'ETH_Forecast', 'DOGE_Forecast'])
print(forecast_df.head())

# Step 5: Obtain Residuals from VAR Model for LSTM Input
residuals = var_model.resid
print(residuals.head())

# Step 6: Visualize Residuals to Confirm Randomness
residuals.plot(figsize=(10,5), title='VAR Residuals')
plt.show()

import pandas as pd

# Ensure 'Date' is the index in both datasets (residuals and sentiment_data)
residuals.index = pd.to_datetime(residuals.index)
sentiment_data.index = pd.to_datetime(sentiment_data.index)

# Align sentiment data to residuals using inner join on the timestamp
lstm_data = residuals.join(sentiment_data, how='inner')

# Check the shape and preview of the resulting dataset
print(lstm_data.shape)
print(lstm_data.head())

# Slice sentiment_data to match the length of lstm_data
sentiment_data = sentiment_data.iloc[-len(lstm_data):]

# Now, lstm_data and sentiment_data should have the same number of rows
print(lstm_data.shape)  # Should show the shape of the combined data
print(sentiment_data.shape)  # Should show the shape of the sentiment data (matching lstm_data)

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

# Add sentiment scores as an additional feature to lstm_data
lstm_data['Sentiment'] = sentiment_data['Sentiment'].iloc[-len(lstm_data):].values

# Check the updated data
print(lstm_data.head())

# Define timesteps (e.g., using past 10 hours for prediction)
timesteps = 10

# Function to create sequences for LSTM input
def create_sequences(data, timesteps):
    X, y = [], []
    data_values = data.values
    for i in range(len(data_values) - timesteps):
        X.append(data_values[i:i+timesteps, :])  # Past timesteps as features
        y.append(data_values[i+timesteps, 0])   # Target variable: BTC residual (1st column)
    return np.array(X), np.array(y)

# Create the sequences for the model
X, y = create_sequences(lstm_data, timesteps)

# Train-test split (80-20 split) with shuffle=False for time series
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle=False)

# Print the shapes of the training and testing sets
print(f'Train shape: {X_train.shape}, Test shape: {X_test.shape}')

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense
from tensorflow.keras.optimizers import Adam

# Define LSTM Model
model = Sequential([
    LSTM(64, activation='relu', input_shape=(timesteps, X.shape[2]), return_sequences=False),  # Input shape
    Dense(1)  # Output layer, predicting the residuals (BTC residuals in this case)
])

# Compile the model with Adam optimizer and MSE loss
model.compile(optimizer=Adam(0.001), loss='mse', metrics=['mae'])

# Display model summary
model.summary()

# Train the model
history = model.fit(
    X_train, y_train,
    epochs=30,  # Set the number of epochs
    batch_size=32,  # Set the batch size
    validation_split=0.2,  # 20% of the data will be used for validation
    verbose=1  # Show training progress
)

test_loss, test_mae = model.evaluate(X_test, y_test)
print(f'Test Loss: {test_loss}')
print(f'Test MAE: {test_mae}')

predictions = model.predict(X_test)

from sklearn.metrics import mean_absolute_percentage_error, mean_squared_error, r2_score, f1_score, roc_auc_score
import numpy as np

# Make predictions
y_pred = model.predict(X_test)

# 1. **MAPE (Mean Absolute Percentage Error)**
mape = mean_absolute_percentage_error(y_test, y_pred)
print(f'MAPE: {mape}')

# 2. **RMSE (Root Mean Squared Error)**
rmse = np.sqrt(mean_squared_error(y_test, y_pred))
print(f'RMSE: {rmse}')

# 3. **MSE (Mean Squared Error)**
mse = mean_squared_error(y_test, y_pred)
print(f'MSE: {mse}')

# 4. **R2 (R-squared Score)**
r2 = r2_score(y_test, y_pred)
print(f'R2: {r2}')

# 5. **Directional Accuracy**
# Directional accuracy checks if the predicted residuals' direction matches the actual residuals' direction.
# We compare if the sign (positive/negative) of the residuals is the same for predicted and actual values.

directional_accuracy = np.mean(np.sign(y_pred) == np.sign(y_test))
print(f'Directional Accuracy: {directional_accuracy}')

# 6. **AUC (Area Under the Curve)**
# To calculate AUC, we need to convert the problem to binary classification.
# If it's a regression problem, you may not compute AUC directly, but here's a sample if binary classification applies.

# AUC is applicable if you convert residuals to classification (positive vs negative)
# This is an approximation for the sake of demonstrating AUC in regression models.
# We'll convert the residuals into binary values (e.g., positive=1, negative=0).

y_test_binary = np.where(y_test > 0, 1, 0)
y_pred_binary = np.where(y_pred > 0, 1, 0)

auc_score = roc_auc_score(y_test_binary, y_pred_binary)
print(f'AUC: {auc_score}')

# 7. **F1 Score**
# The F1 score is typically used for classification, but we can calculate it similarly by comparing the predicted vs actual directions.
f1 = f1_score(y_test_binary, y_pred_binary)
print(f'F1 Score: {f1}')

print(f"MAPE: {mape}")
print(f"RMSE: {rmse}")
print(f"MSE: {mse}")
print(f"R2: {r2}")
print(f"Directional Accuracy: {directional_accuracy}")
print(f"AUC: {auc}")
print(f"F1 Score: {f1_score}")

"""# **XGBOOST-LSTM MODEL BUILDING AND TRAINING**"""

import kagglehub

# Download latest version
path = kagglehub.dataset_download("franoisgeorgesjulien/crypto")

print("Path to dataset files:", path)

import os

dataset_path = "/root/.cache/kagglehub/datasets/franoisgeorgesjulien/crypto/versions/3"
files = os.listdir(dataset_path)
print(files)

import pandas as pd

btc = pd.read_csv(os.path.join(dataset_path, 'Binance_BTCUSDT_1h (1).csv'))
eth = pd.read_csv(os.path.join(dataset_path, 'Binance_ETHUSDT_1h (1).csv'))
doge = pd.read_csv(os.path.join(dataset_path, 'Binance_DOGEUSDT_1h (1).csv'))

print(btc.head())
print(eth.head())
print(doge.head())

btc.head()

eth.head()

doge.head()

from google.colab import files

uploaded = files.upload()  # This will prompt file upload

import pandas as pd

file_path = "crypto-query-tweets1(in).csv"
Twitter_sentiment_df = pd.read_csv(file_path)

# converting with the expected format, handling errors
Twitter_sentiment_df['Date'] = pd.to_datetime(Twitter_sentiment_df['Date'], format="%d/%m/%Y %H:%M", errors='coerce')

# Checked for rows where the conversion failed
print("Rows with conversion issues:")
print(Twitter_sentiment_df[Twitter_sentiment_df['Date'].isna()])

# Now format the column properly
Twitter_sentiment_df['Date'] = Twitter_sentiment_df['Date'].dt.strftime("%Y-%m-%d %H:%M:%S")

# Display the first few rows to check
print(Twitter_sentiment_df.head())

"""# **1.2. Data Cleaning (PRE-PROCESSING)**"""

btcshape = btc.shape
ethshape = eth.shape
dogeshape = doge.shape

print("BTC Shape:", btcshape)
print("ETH Shape:", ethshape)
print("DOGE Shape:", dogeshape)

# Find the minimum shape length
min_length = min(len(btc), len(eth), len(doge))

# Truncate the datasets to the minimum length
btc = btc.iloc[:min_length]
eth = eth.iloc[:min_length]
doge = doge.iloc[:min_length]

# Check the shapes after truncation
print(btc.shape, eth.shape, doge.shape)

import pandas as pd
import numpy as np
import re
from textblob import TextBlob
from sklearn.preprocessing import MinMaxScaler

# Load dataset (assuming it's already a Pandas DataFrame)
df = Twitter_sentiment_df.copy()


#  Drop Unnecessary Columns
df.drop(['username', 'user_location', 'user_description'], axis=1, inplace=True)

#  Handle Missing Values
df.fillna({'followers_count': 0, 'following_count': 0, 'tweet_like_count': 0,
           'tweet_retweet_count': 0, 'tweet_reply_count': 0, 'tweet_quote_count': 0}, inplace=True)

#  Convert 'verified' Column to Binary
df['verified'] = df['verified'].astype(int)

# 4️ Preprocess Tweet Text (Cleaning + Sentiment Analysis)
def clean_text(text):
    text = re.sub(r'http\S+|www\S+', '', text)  # Remove URLs
    text = re.sub(r'@[A-Za-z0-9_]+', '', text)  # Remove mentions
    text = re.sub(r'#[A-Za-z0-9_]+', '', text)  # Remove hashtags
    text = re.sub(r'[^a-zA-Z0-9 ]', '', text)  # Remove special characters
    return text.lower().strip()

def get_sentiment(text):
    return TextBlob(text).sentiment.polarity  # Returns sentiment score (-1 to 1)

df['clean_tweet'] = df['tweet_text'].astype(str).apply(clean_text)
df['sentiment_score'] = df['clean_tweet'].apply(get_sentiment)

# Convert 'Date' Column to Datetime & Extract Features
df['Date'] = pd.to_datetime(df['Date'])
df['hour'] = df['Date'].dt.hour
df['day'] = df['Date'].dt.day
df['weekday'] = df['Date'].dt.weekday  # 0 = Monday, 6 = Sunday

# Convert 'Date' Column to Datetime & Extract Features
df['Date'] = pd.to_datetime(df['Date'])
df['hour'] = df['Date'].dt.hour
df['day'] = df['Date'].dt.day
df['weekday'] = df['Date'].dt.weekday  # 0 = Monday, 6 = Sunday


# Normalize Numeric Features
scaler = MinMaxScaler()
numeric_cols = ['followers_count', 'following_count', 'tweet_like_count',
                'tweet_retweet_count', 'tweet_reply_count', 'tweet_quote_count', 'sentiment_score']
df[numeric_cols] = scaler.fit_transform(df[numeric_cols])

# Show all columns in the dataset
pd.set_option('display.max_columns', None)  # Ensure all columns are visible
df.head(5)  # Display first few rows

btc.head()

# Remove 'Symbol' column from the 'btc' dataset
btc = btc.drop(columns=['Symbol'])

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score
import seaborn as sns

# Ensure the 'Date' columns are datetime type
btc['Date'] = pd.to_datetime(btc['Date'])
df['Date'] = pd.to_datetime(df['Date'])

# Merge the datasets on 'Date'
data = pd.merge(btc, df[['Date', 'sentiment_score']], on='Date', how='inner')

# Check for missing values
print(data.isnull().sum())

# Fill or drop missing values if necessary
data.fillna(method='ffill', inplace=True)  # Forward fill for simplicity


# Visualize data distribution
plt.figure(figsize=(10,6))
sns.heatmap(data.corr(), annot=True, cmap='coolwarm', fmt='.2f')
plt.show()

# Scaling the 'Open', 'High', 'Low', 'Close', 'Volume' columns for LSTM
scaler = MinMaxScaler(feature_range=(0, 1))
scaled_data = scaler.fit_transform(data[['Open', 'High', 'Low', 'Close', 'Volume BTC', 'Volume USDT', 'sentiment_score']])

# Add the scaled data back to the dataframe
scaled_df = pd.DataFrame(scaled_data, columns=['Open', 'High', 'Low', 'Close', 'Volume BTC', 'Volume USDT', 'sentiment_score'])
scaled_df['Date'] = data['Date']
scaled_df.set_index('Date', inplace=True)

# Prepare the data for time-series forecasting
# Function to create sequences for the LSTM model
def create_sequences(df, n_steps):
    X, y = [], []
    for i in range(n_steps, len(df)):
        X.append(df.iloc[i-n_steps:i].values)  # n_steps previous data points
        y.append(df.iloc[i, 3])  # 'Close' column as the target
    return np.array(X), np.array(y)

# Set the number of previous time steps to consider
n_steps = 24  # You can adjust this as per the dataset

# Create sequences
X, y = create_sequences(scaled_df, n_steps)

# Split the data into training and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle=False)

print("Data Preprocessing completed!")

import numpy as np
import xgboost as xgb
from keras.models import Sequential
from keras.layers import LSTM, Dense
from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_percentage_error

# XGBoost model for feature selection
xgb_model = xgb.XGBRegressor(objective='reg:squarederror')
xgb_model.fit(X_train.reshape(X_train.shape[0], -1), y_train)

# Get feature importance from XGBoost
xgb_importance = xgb_model.feature_importances_

# LSTM model to predict based on XGBoost features
lstm_model = Sequential()
lstm_model.add(LSTM(50, return_sequences=True, input_shape=(X_train.shape[1], X_train.shape[2])))
lstm_model.add(LSTM(50))
lstm_model.add(Dense(1))
lstm_model.compile(optimizer='adam', loss='mean_squared_error')

# Fit the LSTM model
lstm_model.fit(X_train, y_train, epochs=10, batch_size=32, validation_data=(X_test, y_test))

# Predict using the XGBoost-LSTM hybrid model
xgb_preds = xgb_model.predict(X_test.reshape(X_test.shape[0], -1))
lstm_preds = lstm_model.predict(X_test)

# Combine predictions (simple average of both models)
hybrid_preds = 0.5 * (xgb_preds + lstm_preds.flatten())

# Evaluate model performance
rmse = np.sqrt(mean_squared_error(y_test, hybrid_preds))
r2 = r2_score(y_test, hybrid_preds)
mape = mean_absolute_percentage_error(y_test, hybrid_preds)

# Directional Accuracy function
def directional_accuracy(y_true, y_pred):
    # Remove the last element to align with the diff output
    y_true_diff = np.diff(y_true)
    y_pred_diff = np.diff(y_pred)

    # Calculate the direction (sign) of changes
    direction_true = np.sign(y_true_diff)
    direction_pred = np.sign(y_pred_diff)

    # Compare the directions (up or down)
    accuracy = np.sum(direction_true == direction_pred) / len(direction_true)
    return accuracy * 100  # Return percentage

# Calculate Directional Accuracy
directional_accuracy_score = directional_accuracy(y_test, hybrid_preds)


# Visualize predictions
plt.figure(figsize=(10,6))
plt.plot(y_test, label='Actual', color='blue')
plt.plot(hybrid_preds, label='Predicted', color='red')
plt.title('XGBoost-LSTM Model: Actual vs Predicted')
plt.legend()
plt.show()

#Print performance metrics
print(f"XGBoost-LSTM Model RMSE: {rmse}")
print(f"R2 Score: {r2}")
print(f"MAPE: {mape}")
print(f"directional Accuracy: {directional_accuracy_score:.2f}%")

"""# **CNN-LSTM MODEL BUILDING AND TRAINING**"""

from keras.layers import Conv1D, MaxPooling1D

# CNN-LSTM model
cnn_lstm_model = Sequential()
cnn_lstm_model.add(Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(X_train.shape[1], X_train.shape[2])))
cnn_lstm_model.add(MaxPooling1D(pool_size=2))
cnn_lstm_model.add(LSTM(50))
cnn_lstm_model.add(Dense(1))
cnn_lstm_model.compile(optimizer='adam', loss='mean_squared_error')

# Fit the CNN-LSTM model
cnn_lstm_model.fit(X_train, y_train, epochs=10, batch_size=32, validation_data=(X_test, y_test))

# Predict using the CNN-LSTM model
cnn_lstm_preds = cnn_lstm_model.predict(X_test)

# Visualize predictions
plt.figure(figsize=(10,6))
plt.plot(y_test, label='Actual', color='blue')
plt.plot(cnn_lstm_preds, label='Predicted', color='green')
plt.title('CNN-LSTM Model: Actual vs Predicted')
plt.legend()
plt.show()

import numpy as np
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Conv1D, MaxPooling1D, LSTM, Dense
from sklearn.metrics import mean_absolute_percentage_error, mean_squared_error, r2_score

# CNN-LSTM model
cnn_lstm_model = Sequential()
cnn_lstm_model.add(Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(X_train.shape[1], X_train.shape[2])))
cnn_lstm_model.add(MaxPooling1D(pool_size=2))
cnn_lstm_model.add(LSTM(50))
cnn_lstm_model.add(Dense(1))
cnn_lstm_model.compile(optimizer='adam', loss='mean_squared_error')

# Fit the CNN-LSTM model
cnn_lstm_model.fit(X_train, y_train, epochs=10, batch_size=32, validation_data=(X_test, y_test))

# Predict using the CNN-LSTM model
cnn_lstm_preds = cnn_lstm_model.predict(X_test)

# Flatten the predictions to match the shape of y_test
cnn_lstm_preds = cnn_lstm_preds.flatten()

# Calculate MAPE (Mean Absolute Percentage Error)
mape = mean_absolute_percentage_error(y_test, cnn_lstm_preds)

# Calculate RMSE (Root Mean Squared Error)
rmse = np.sqrt(mean_squared_error(y_test, cnn_lstm_preds))

# Calculate R² (R-squared score)
r2 = r2_score(y_test, cnn_lstm_preds)

# Directional Accuracy function
def directional_accuracy(y_true, y_pred):
    # Remove the last element to align with the diff output
    y_true_diff = np.diff(y_true)
    y_pred_diff = np.diff(y_pred)

    # Calculate the direction (sign) of changes
    direction_true = np.sign(y_true_diff)
    direction_pred = np.sign(y_pred_diff)

    # Compare the directions (up or down)
    accuracy = np.sum(direction_true == direction_pred) / len(direction_true)
    return accuracy * 100  # Return percentage

# Calculate Directional Accuracy
directional_accuracy_score = directional_accuracy(y_test, cnn_lstm_preds)


# Visualize predictions
plt.figure(figsize=(10,6))
plt.plot(y_test, label='Actual', color='blue')
plt.plot(cnn_lstm_preds, label='Predicted', color='green')
plt.title('CNN-LSTM Model: Actual vs Predicted')
plt.legend()
plt.show()

print(f"CNN-LSTM Model MAPE: {mape}")
print(f"CNN-LSTM Model RMSE: {rmse}")
print(f"CNN-LSTM Model R²: {r2}")
print(f"CNN-LSTM Model Directional Accuracy: {directional_accuracy*100:.2f}%")

