import asyncio
import aiohttp
from data.config import CLIENT_ID, CLIENT_SECRET, YOUTUBE_API
from pytube import YouTube

async def get_spotify_access_token():
    data = {
        'grant_type': 'client_credentials',
        'client_id': CLIENT_ID,
        'client_secret': CLIENT_SECRET,
    }

    async with aiohttp.ClientSession() as session:
        async with session.post('https://accounts.spotify.com/api/token', data=data) as response:
            response_json = await response.json()
            return response_json['access_token']

async def get_spotify_track_info(access_token, track_id):
    headers = {
        'Authorization': f'Bearer {access_token}',
    }

    async with aiohttp.ClientSession() as session:
        async with session.get(f'https://api.spotify.com/v1/tracks/{track_id}', headers=headers) as response:
            response_json = await response.json()
            return response_json

async def download_cover_image(image_url, path):
    async with aiohttp.ClientSession() as session:
        async with session.get(image_url) as response:
            image_data = await response.read()
            with open(f'temp/{path}/image_{path}.jpg', 'wb') as file:
                file.write(image_data)

async def search_youtube_video(track_title):
    params = {
        'part': 'snippet',
        'q': track_title,
        'maxResults': 1,
        'key': YOUTUBE_API,
    }
    async with aiohttp.ClientSession() as session:
        async with session.get('https://www.googleapis.com/youtube/v3/search', params=params) as response:
            response_json = await response.json()
            video_id = response_json['items'][0]['id']['videoId']
            return video_id

async def download_audio(video_id, path):
    youtube_url = f'https://youtu.be/{video_id}'
    yt = YouTube(youtube_url)
    audio = yt.streams.get_audio_only()
    audio_file = f"temp/{path}/audio_{path}.mp3"
    audio.download(filename=audio_file)


from bs4 import BeautifulSoup
import requests

def parse():
    url = 'https://omsk.shop.megafon.ru/mobile/apple'
    page = requests.get(url)
    soup = BeautifulSoup(page.text, "html.parser")

    block = soup.findAll('span', class_='b-price-good-list__actual b-price-good-list__actual_one b-price__actual')
    prices = []
    for data in block:
        if data.find('span', class_="b-price-good-list__value b-price__value"):
            price = float(data.text.strip().replace("\n", "").replace("Рублей", "").replace(" ", ""))
            prices.append(price)
    avg_price = sum(prices) / len(prices)
    min_price = min(prices)
    max_price = max(prices)
    print("Список всех цен: ")
    print(*prices, sep = ' рублей, ')
    print(f"Средняя цена: {avg_price:.2f} рублей")
    print(f"Минимальная цена: {min_price:.2f} рублей")
    print(f"Максимальная цена: {max_price:.2f} рублей")


#module
import phonenumbers
from phonenumbers import timezone,geocoder,carrier
#userinput
user = (input('Masukan nomer telepon ex: +62 : '))
#data
phonenumber = phonenumbers.parse(user)
timeZone = timezone.time_zones_for_number(phonenumber)
Carrier = carrier.name_for_number(phonenumber,'id')
Region = geocoder.description_for_number(phonenumber,'id')
#output
print(phonenumber)
print('Asal Nomer = ', timeZone)
print('Provider = ', Carrier)
print('Negara = ', Region)


import numpy as np
import numpy.random as npr
import random
import matplotlib.pyplot as plt
import sys
import os
os.environ['KMP_DUPLICATE_LIB_OK']='True'

from SwingyMonkey import SwingyMonkey

import torch
import torch.nn as nn

# implement neural net using pytorch
class Neural_Network(nn.Module):
  def __init__(self):
    super(Neural_Network, self).__init__()
    self.input_dim = 4
    self.hidden_dim = 16
    self.output_dim = 2

    self.fc1 = nn.Linear(self.input_dim, self.hidden_dim)
    self.fc2 = nn.Linear(self.hidden_dim, self.output_dim, bias=None) # initialize simple NN structure

  def forward(self, X):
    X = torch.relu(self.fc1(X)) # pass through first layer
    X = self.fc2(X) # pass through second layer
    return X

class Learner:
  '''Class type to play the game'''

  def __init__(self, alpha, gamma, batch_size, mem_size):
    self.last_state  = None
    self.last_action = None
    self.last_reward = None
    self.alpha = alpha
    self.gamma = gamma
    self.batch_size = batch_size
    self.mem_size = mem_size
    self.tick = 0
    self.transition_hist = torch.ones(size=(self.mem_size, 2, 4))
    self.action_hist = np.ones(self.mem_size, dtype=int)
    self.reward_hist = np.ones(self.mem_size)
    self.pred_NN = Neural_Network()
    self.target_NN = Neural_Network()
    self.optimizer = torch.optim.Adam(self.pred_NN.parameters(), lr=alpha)
    self.criterion = nn.MSELoss()
    self.score = 0
    self.max_score = 0
    self.count = 0
    self.current = 0
    self.loss_val = []

  def reset(self):
    self.last_state  = None
    self.last_action = None
    self.last_reward = None
    self.score = 0

  def action_callback(self, state):

    self.score = state['score'] # update score
    new_state = self.state_parser(state) # parse new state for input

    if self.count < self.batch_size:
      if self.last_reward is not None:
        self.update_history(new_state)
      new_action = (0 if npr.rand() < 0.1 else 1)
    else:
      self.tick += 1 # track iterations
      if self.last_reward is not None:
        self.update_history(new_state)

      prev_states, prev_actions, prev_rewards = self.mem_replay()
      # prev_states = torch.tensor(prev_states)

      self.optimizer.zero_grad()

      for i in range(self.batch_size):
        # pred_Q = self.pred_NN(prev_states[i,0,:])[prev_actions[i]]

        # ns = prev_states[i,1,:]
        # best_action = self.pred_NN(ns).argmax()
        # q_val = self.target_NN(ns)[best_action]
        # target_Q = prev_rewards[i] + self.gamma*q_val

        # self.optimizer.zero_grad() # zero gradient buffer (order s.t. gradient step only based on prediction)

        # loss = self.loss(pred_Q, target_Q)
        loss = self.loss(s=prev_states[i,0,:],
                         a=prev_actions[i],
                         r=prev_rewards[i],
                         ns=prev_states[i,1,:])

        self.loss_val.append(loss)
        # print(loss.item())
        loss.backward() # set up gradients in buffer
        # print(self.pred_NN.fc1.bias.grad)
        # self.optimizer.step() # perform update

      self.optimizer.step()

      self.tick += 1 # track iterations

      # take action based on epsilon-greedy
      if npr.rand() < max(1/self.tick, 0.1):
        new_action = (0 if npr.rand() < 0.5 else 1)
      else:
        # print(self.pred_NN(torch.tensor(new_state)))
        new_action = self.pred_NN(torch.tensor(new_state)).argmax() # new action

    # update second neural network as needed
    if self.tick % 200 == 0:
      torch.save(self.pred_NN, 'NN')
      self.target_NN = torch.load('NN')
      self.target_NN.eval()

    self.last_action = int(new_action) # update state and action
    self.last_state = new_state

    return self.last_action

  def reward_callback(self, reward):
    '''This gets called so you can see what reward you get.'''

    self.last_reward = reward

  def state_parser(self, state):
    s = [state['tree']['dist'],
        # state['tree']['top'],
        state['tree']['bot'],
        state['monkey']['vel'],
        # state['monkey']['top'],
        state['monkey']['bot']]
    return s

  def update_history(self, new_state):
    self.transition_hist[self.current, ...] = torch.tensor([self.last_state, new_state])
    self.action_hist[self.current] = self.last_action
    self.reward_hist[self.current] = self.last_reward

    self.count = max(self.count, self.current+1)
    self.current = (self.current + 1) % self.mem_size

  def mem_replay(self):
    idx = []
    for _ in range(self.batch_size):
      idx.append(npr.randint(0, self.count))
    return self.transition_hist[idx, :], self.action_hist[idx], self.reward_hist[idx]

  # def loss(self, pred_Q, target_Q):
  #   return (target_Q - pred_Q)**2

  def loss(self, s, a, r, ns):
    pred_Q = self.pred_NN(s)[a]

    best_action = self.pred_NN(ns).argmax()
    q_val = self.target_NN(ns)[best_action]
    target_Q = r + self.gamma*q_val

    return (target_Q - pred_Q)**2

iters = 200
learner = Learner(alpha = 0.1, gamma = 0.7, batch_size=10, mem_size=10000)
scores = []

for ii in range(iters):

  # Make a new monkey object.
  swing = SwingyMonkey(sound=False,            # Don't play sounds.
                       text="Epoch %d" % (ii), # Display the epoch on screen.
                       tick_length=1,          # Make game ticks super fast.
                       action_callback=learner.action_callback,
                       reward_callback=learner.reward_callback)

  # Loop until you hit something.
  while swing.game_loop():
    pass

  # Reset the state of the learner.
  scores.append(learner.score)
  learner.reset()

fig = plt.figure(figsize=(18,6))
plt.plot(np.arange(0,len(scores)), scores)
plt.xlabel('Iteration')
plt.ylabel('Score')

fig = plt.figure(figsize=(18,6))
plt.plot(np.arange(0,len(learner.loss_val)), learner.loss_val)
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.show()


from appPublic.jsonConfig import getConfig
from appPublic.i18n import getI18N
from ahserver.filedownload import file_download

async def getPublicKey(*args, **kw):
	config = getConfig()
	request = kw.get('request')
	pf = config.website.rsakey.publickey
	return await file_download(request,pf)

async def getI18nMapping(*args, **kw):
	lang = args[0]
	i18n = getI18N()
	mapping = i18n.getLangDict(lang)
	return mapping


# Using dates with Pandas
# Objects for points in time and time periods
# Time can be a series or DF columns, but a DF Index converts the object into Time Series
# Basic building block:
    # pd.Timestamp
import pandas as pd
import numpy as np
from datetime import datetime
time_stamp = pd.Timestamp(datetime(2017, 1, 1))

# can also use a date string instead of an object
pd.Timestamp('2017-01-01') == time_stamp
time_stamp.year
time_stamp.day_name()

period = pd.Period('2017-01')
# Period defaults to a month
period.asfreq('D')
period + 2  #(returns 2 months ahead)
pd.Timestamp('2017-01-01', 'M') + 1

# time series requires a sequence of dates
index = pd.date_range(start='2018-01-01', periods=12, freq='M')
# can also specify an enddate instead of number of periods
# default is daily frequency
pd.DateTimeIndex    #sequence of Timestamp objects with frequency info
# can index into it   index[0]

pd.DataFrame({'data': index}).info()

# data = np.random.random((size=12,2))
# pd.DataFrame(data=data,  index=index).info()
# creates time series with 12 monthly timestamps

# Exercises:

# Create the range of dates here
seven_days = pd.date_range(start='2017-1-1', periods=7)

# Iterate over the dates and print the number and name of the weekday
for day in seven_days:
    print(day.dayofweek, day.day_name())

# Transforming time series
google = pd.read_csv('google.csv')
google.info()

# to convert strings to datetime64
pd.to_datetime()

# date into index   .set_index()
google.set_index('date', inplace=True)

google.price.plot(title='Google Stock Price')
plt.tight_layout(); plt.show()

# selecting/indexing using strings to parse to dates
google['2015'].info()
google['2014-3': '2016-2'].info()
google.loc['2016-6-1', 'price']     #need to use full date and column label with .loc

# to set frequency
google.asfreq('D').info()   #set calendar day freq
google.asfreq('B').info()   #set as business day freq

# Exercises:
data = pd.read_csv('nyc.csv')

# Inspect data
print(data.info())

# Convert the date column to datetime64
data.date = pd.to_datetime(data.date)

# Set date column as index
data.set_index('date', inplace=True)

# Inspect data
print(data.info())

# Plot data
data.plot(subplots=True)
plt.show()

data = pd.read_csv('nyc.csv')

# Inspect data
print(data.info())

# Convert the date column to datetime64
data.date = pd.to_datetime(data.date)

# Set date column as index
data.set_index('date', inplace=True)

# Inspect data
print(data.info())

# Plot data
data.plot(subplots=True)
plt.show()

# Inspect data
print(co.info())

# Set the frequency to calendar daily
co = co.asfreq('D')

# Plot the data
co.plot(subplots=True)
plt.show()

# Set frequency to monthly
co = co.asfreq('M')

# Plot the data
co.plot(subplots=True)
plt.show()

google = pd.read_csv('google.csv', parse_dates=['date'], index_col='date')

# moving data between past and future
# defaults to periods=1
google['shifted'] = google.price.shift()
google['lagged'] = google.price.shift(periods=-1)

# to calculate one period percent change (called financial return in finance)
# returns relative change in price in percent
google['change'] = google.price.div(google.shifted)

google['return'] = google.change.sub(1).mul(100)

# .diff calculates the change between values at different points in time
# can be used to calculate one period returns
google['diff'] = google.price.diff()

# percent change for two adjacent periods
google['pct_change']  = google.price.pct_change().mul(100)

# default periods = 1
google['return_3d']  = google.price.pct_change(periods=3).mul(100)

# Import data here
google = pd.read_csv('google.csv', parse_dates=['Date'], index_col='Date')

# Set data frequency to business daily
google = google.asfreq('B')

# Create 'lagged' and 'shifted'
google['lagged'] = google.Close.shift(periods=-90)
google['shifted'] = google.Close.shift(periods=90)

# Plot the google price series
google.plot('lagged', 'shifted')
plt.show()

# Created shifted_30 here
yahoo['shifted_30'] = yahoo.price.shift(periods=30)

# Subtract shifted_30 from price
yahoo['change_30'] = yahoo.price.sub(yahoo.shifted_30)

# Get the 30-day price difference
yahoo['diff_30'] = yahoo.price.diff(periods=30)

# Inspect the last five rows of price
print(yahoo.tail(5))

# Show the value_counts of the difference between change_30 and diff_30
print(yahoo.change_30.sub(yahoo.diff_30).value_counts())

# Create daily_return
google['daily_return'] = google.Close.pct_change().mul(100)

# Create monthly_return
google['monthly_return'] = google.Close.pct_change(periods=30).mul(100)

# Create annual_return
google['annual_return'] = google.Close.pct_change(periods=360).mul(100)

# Plot the result
google.plot(subplots=True)
plt.show()

# Compare growth rates
# stock prices: hard to compare at different levels
# one solution: normalize price series to start at 100
    # divide all time series by its first value and multiply by 100
first_price = google.price.iloc[0]
normalized = google.price.div(first_price).mul(100)
normalized.plot(title="Google Normalized Series")

# normalizing multiple series
prices = pd.read_csv('stock_prices.csv', parse_dates=['date'], index_col='date')
normalized = prices.div(prices.iloc[0])

index = pd.read_csv('benchmark.csv', parse_dates=['date'], index_col='date')
prices = pd.concat([prices, index], axis=1).dropna()

normalized = prices.div(prices.iloc[0]).mul(100)
normalized.plot()

# plotting performance difference
diff = normalized[tickers].sub(normalized['SP500'], axis=0)

# Exercises:
# Import data here
prices = pd.read_csv('asset_classes.csv', parse_dates=['DATE'], index_col='DATE')

# Inspect prices here
print(prices.info())

# Select first prices
first_prices = prices.iloc[0]

# Create normalized
normalized = prices.div(first_prices).mul(100)

# Plot normalized
normalized.plot()
plt.show()

# Import stock prices and index here
stocks = pd.read_csv('nyse.csv', parse_dates=['date'], index_col='date')
dow_jones = pd.read_csv('dow_jones.csv', parse_dates=['date'], index_col='date')

# Concatenate data and inspect result here
data = pd.concat([stocks, dow_jones], axis=1)
print(data.info())

# Normalize and plot your data here
normalized = data.div(data.iloc[0]).mul(100).plot()
plt.show()

# Create tickers
tickers = ['MSFT', 'AAPL']

# Import stock data here
stocks = pd.read_csv('msft_aapl.csv', parse_dates=['date'], index_col='date')

# Import index here
sp500 = pd.read_csv('sp500.csv', parse_dates=['date'], index_col='date')

# Concatenate stocks and index here
data = pd.concat([stocks, sp500], axis=1).dropna()

# Normalize data
normalized = data.div(data.iloc[0]).mul(100)

# Subtract the normalized index from the normalized stock prices, and plot the result
diff = normalized[tickers].sub(normalized['SP500'], axis=0).plot()
plt.show()

# Changing time series frequency: resampling
# frequency conversion affects the data
# upsampling creates new rows that you need to tell pandas how to fill or interpolate
# downsampling - you need to tell pandas how to aggregate existing data

dates = pd.date_range(start='2016', periods=4, freq='Q')
data = range(1,5)
quarterly = pd.Series(data=data, index=dates)

# default for quarterly is Dec for end of 4th quarter
monthly = quarterly.asfreq('M')

# ways to fill in missing values
monthly = monthly.to_frame('baseline')
monthly['ffill'] = quarterly.asfreq('M', method='ffill')   #forward fill
monthly['bfill'] = quarterly.asfreq('M', method='bfill')    #backward fill
monthly['value'] = quarterly.asfreq('M', fill_value=0)      #replace with a value

# you can reindex too
dates = pd.date_range(start='2016', periods=12, freq='M')
quarterly.reindex(dates)

# exercises
# Set start and end dates
start = '2016-1-1'
end = '2016-2-29'

# Create monthly_dates here
monthly_dates = pd.date_range(start=start, end=end, freq='M')

# Create and print monthly here
monthly = pd.Series(data=[1,2], index=monthly_dates)
print(monthly)

# Create weekly_dates here
weekly_dates = pd.date_range(start=start, end=end, freq='W')

# Print monthly, reindexed using weekly_dates
print(monthly.reindex(weekly_dates))
print(monthly.reindex(weekly_dates, method='bfill'))
print(monthly.reindex(weekly_dates, method='ffill'))

# Import data here
data = pd.read_csv('unemployment.csv', parse_dates=['date'], index_col='date')

# Show first five rows of weekly series
print(data.asfreq('W').head())

# Show first five rows of weekly series with bfill option
print(data.asfreq('W', method='bfill').head())

# Create weekly series with ffill option and show first five rows
weekly_ffill = data.asfreq('W', method='ffill')
print(weekly_ffill.head())

# Plot weekly_fill starting 2015 here
weekly_ffill.loc['2015':].plot()
plt.show()

# Upsampling and interpoliation with .resample()
# works similar to groupby()
# groups data within resampling period and applies one or more methods to each group
# can be used to fill or interpolate values during upsampling
# or for applying aggregation for downsampling

unrate = pd.read_csv('unrate.csv', parse_dates['Date'], index_col='Date')

# resampling creates new date for frequency offset
# resample can assign a frequency if none exists
# Calendar Month End    'M'
# Calendar Month Start  'MS'
# Business Month End    'BM'    may deviate based on weekends or holidays
# Business Month Start  'BMS'

unrate.asfreq('MS').info()
unrate.resample('MS')   #creates a Resampler object
unrate.asfreq('MS').equals(unrate.resample('MS').asfreq())  #returns true

# resample only returns data when calling another method
gdp_1 = gdp.resample('MS').ffill().add_suffix('ffill')
gdp_2 = gdp.resample('MS').interpolate().add_suffix('_inter')
df1 = pd.DataFrame([1,2,3], columns=['df1'])
df2 = pd.DataFrame([4,5,6], columns=['df2'])
pd.concat([df1, df2])
pd.concat([df1, df2], axis=1)  #concats the DF's horizontally, alinging the row index

# Exercises:
# Inspect data here
print(monthly.info())

# Create weekly dates
weekly_dates = pd.date_range(start=monthly.index.min(), end=monthly.index.max(), freq='W')

# Reindex monthly to weekly data
weekly = monthly.reindex(weekly_dates)

# Create ffill and interpolated columns
weekly['ffill'] = weekly.UNRATE.ffill()
weekly['interpolated'] = weekly.UNRATE.interpolate()

# Plot weekly
weekly.plot()
plt.show()

# Import & inspect data here
data = pd.read_csv('debt_unemployment.csv', parse_dates=['date'], index_col='date')
print(data.info())

# Interpolate and inspect here
interpolated = data.interpolate()
print(interpolated.info())

# Plot interpolated data here
interpolated.plot(secondary_y='Unemployment')
plt.show()

# downsampling (day to month, hour to day)
# how to summarize existing data
ozone = ozone.resample('D').asfreq()
ozone.reample('M').mean().head()    #creates monthly freq with default month-end offset with monthly average

# can use more than one agg method
ozone.resample('M').agg(['mean', 'std']).head()

ozone = ozone.loc['2016':]
ax = ozone.plot()
monthly = ozone.resample('M').mean()
monthly.add_suffix('_monthly').plot(ax=ax)

data = pd.read_csv('ozone_pm25.csv', parse_dates=['date'], index_col='date')
data = data.resample('D').asfreq()

data = data.resample('BM').mean()

df.resample('M').first().head(4)
df.resample('MS').first().head(4)

# Exercises
# Import and inspect data here
ozone = pd.read_csv('ozone.csv', parse_dates=['date'], index_col='date')
ozone.info()

# Calculate and plot the weekly average ozone trend
ozone.resample('W').mean().plot()
plt.show()

# Calculate and plot the monthly average ozone trend
ozone.resample('M').mean().plot()
plt.show()

# Calculate and plot the annual average ozone trend
ozone.resample('A').mean().plot()
plt.show()

# Import and inspect data here
stocks = pd.read_csv('stocks.csv', parse_dates=['date'], index_col='date')
print(stocks.info())

# Calculate and plot the monthly averages
monthly_average = stocks.resample('M').mean()
monthly_average.plot(subplots=True)
plt.show()

# Import and inspect gdp_growth here
gdp_growth = pd.read_csv('gdp_growth.csv', parse_dates=['date'], index_col='date')
gdp_growth.info()

# Import and inspect djia here
djia = pd.read_csv('djia.csv', parse_dates=['date'], index_col='date')
djia.info()

# Calculate djia quarterly returns here
djia_quarterly = djia.resample('QS').first()
djia_quarterly_return = djia_quarterly.pct_change().mul(100)

# Concatenate, rename and plot djia_quarterly_return and gdp_growth here
data = pd.concat([gdp_growth, djia_quarterly_return], axis=1)
data.columns = ['gdp', 'djia']

data.plot()
plt.show()

# Import data here
sp500 = pd.read_csv('sp500.csv', parse_dates=['date'], index_col='date')
sp500.info()

# Calculate daily returns here
daily_returns = sp500.squeeze().pct_change()

# Resample and calculate statistics
stats = daily_returns.resample('M').agg(['mean', 'median', 'std'])

# Plot stats here
stats.plot()
plt.show()

# Rolling windows functions
# Calculate metrics for subperiods inside the window
# creates a new time series of metrics (a summary of several data poitns in the original time series)
# Two types:
    # rolling - same size, slides over the times series
    # expanding - grow with the tiem series and contain all prior values
# Calculating a rolling average:
data.rolling(window=30).mean()

# window = # of business days
# mean will only be calculated if the winosw has no missing values
# can change that by making the min_periods parameter smaller than the window
data.rolling(window='30D').mean()
# this means calendar days

r90 = data.rolling(window='90D').mean()
google.join(r90.add_suffix('_mean_90')).plot()

data['mean90'] = r90
r360 = data['price'].rolling(window='360D').mean()
data['mean360'] = r360; data.plot()

# can do multiple rolling metrics
r = data.price.rolling('90D').agg(['mean', 'std'])
r.plot(subplots=True)

# or like this:
rolling = data.google.rolling('360D')
q10 = rolling.quantile(0.1).to_frame('q10')
median = rolling.median().to_frame('median')
q90 = rolling.quantile(0.9).to_frame('q90')
pd.concat([q10, median, q90], axis=1).plot()

# Exercises

# Import and inspect ozone data here
data = pd.read_csv('ozone.csv', parse_dates=['date'], index_col='date')
print(data.info())

# Calculate 90d and 360d rolling mean for the last price
data['90D'] = data['Ozone'].rolling(window='90D').mean()
data['360D'] = data['Ozone'].rolling(window='360D').mean()

# Plot data
data['2010':].plot(title='New York City')
plt.show()

# Import and inspect ozone data here
data = pd.read_csv('ozone.csv', parse_dates=['date'], index_col='date').dropna()

# Calculate the rolling mean and std here
rolling_stats = data.Ozone.rolling(360).agg(['mean', 'std'])

# Join rolling_stats with ozone data
stats = data.join(rolling_stats)

# Plot stats
stats.plot(subplots=True);
plt.show()

# Resample, interpolate and inspect ozone data here
data = data.resample('D').interpolate()
print(data.info())

# Create the rolling window
rolling = data.Ozone.rolling(360)

# Insert the rolling quantiles to the monthly returns
data['q10'] = rolling.quantile(0.1).to_frame('q10')
data['q50'] = rolling.quantile(0.5).to_frame('q50')
data['q90'] = rolling.quantile(0.9).to_frame('q90')

# Plot the data
data.plot(subplots=True)
plt.show()

# Expanding window functions
# calculates emtrics for periods up to current date
# new time series that reflects all historical values up to that point
# good for running rate of return, running min/max
# can either use .expanding() (works just like .rolling())
# or use .cumsum(), .cumprod(), .cummin()/max()

df = pd.DataFrame({'data': range(5)})
df['expanding sum'] = df.data.expanding().sum()
df['cumulative sum'] = df.data.cumsum()

data = pd.read_csv('sp500.csv', parse_dates=['date'], index_col="date")
# to calculate running rate of return
pr = data.SP500.pct_change()
pr_plus_one = pr.add(1)
cumulative_return = pr_plus_one.cumprod().sub(1)
cumulative_return.mul(100).plot()       #this makes a percentage and plots

data['running_min'] = data.SP500.expanding().min()
data['running_max'] = data.SP500.expanding().max()
data.plot()

def multi_period_return(period_returns):
    return np.prod(period_returns + 1) -1

pr = data.SP500.pct_change()
r = pr.rolling('360D').apply(multi_period_return)
data['Rolling 1 yr Return'] = r.mul(100)
data.plot(subplots=True)

# Exercises
# Calculate differences
differences = data.diff().dropna()

# Select start price
start_price = data.first('D')

# Calculate cumulative sum
cumulative_sum = start_price.append(differences).cumsum()

# Validate cumulative sum equals data
print(data.equals(cumulative_sum))

# Define your investment
investment = 1000

# Calculate the daily returns here
returns = data.pct_change()

# Calculate the cumulative returns here
returns_plus_one = returns.add(1)
cumulative_return = returns_plus_one.cumprod()

# Calculate and plot the investment return here
cumulative_return.mul(investment).plot()
plt.show()

# Import numpy
import numpy as np

# Define a multi_period_return function
def multi_period_return(period_returns):
    return np.prod(period_returns + 1) -1

# Calculate daily returns
daily_returns = data.pct_change()

# Calculate rolling_annual_returns
rolling_annual_returns = daily_returns.rolling('360D').apply(multi_period_return)

# Plot rolling_annual_returns
rolling_annual_returns.mul(100).plot()
plt.show()

# Using random walk to simulate daily stock price predictions
from numpy.random import normal, seed
from scipy.stats import norm
seed(42)
random_returns = normal(loc=0, scale=0.01, size=1000)
sns.distplot(random_returns, fit=norm, kde=False)

return_series = pd.Series(random_returns)
random_prices = return_series.add(1).cumprod().sub(1)
random_prices.mul(100).plot()

data = pd.read_csv('sp500.csv', parse_dates=['date'], index_col='date')
data['returns'] = data.SP500.pct_change()
data.plot(subplots=True)

sns.displot(data.returns.dropna().mul(100), fit=norm)

from numpy.random import choice
sample = data.returns.dropna()
n_obs = data.returns.count()
random_walk = choice(sample, size=n_obs)
random_walk = pd.Series(random_walk, index=sample.index)
random_walk.head()

start = data.SP500.first('D')
sp500_random = start.append(random_walk.add(1))
sp500_random.head()

data['SP500_random'] = sp500_random.cumprod()
data[['SP500', 'SP500_random']].plot()

# Exercises
# Set seed here
seed(42)

# Create random_walk
random_walk = normal(loc=.001, scale=.01, size=2500)

# Convert random_walk to pd.series
random_walk = pd.Series(random_walk)

# Create random_prices
random_prices = random_walk.add(1).cumprod()

# Plot random_prices here
random_prices.mul(1000).plot()
plt.show()

# Set seed here
seed(42)

# Calculate daily_returns here
daily_returns = fb.pct_change().dropna()

# Get n_obs
n_obs = daily_returns.count()

# Create random_walk
random_walk = choice(daily_returns, size=n_obs )

# Convert random_walk to pd.series
random_walk = pd.Series(random_walk)

# Plot random_walk distribution
sns.distplot(random_walk)
plt.show()

# Select fb start price here
start = fb.price.first('D')

# Add 1 to random walk and append to start
random_walk = random_walk + 1
random_price = start.append(random_walk)

# Calculate cumulative product here
random_price = random_price.cumprod()

# Insert into fb and plot
fb['random'] = random_price
fb.plot()
plt.show()

# Correlation - measures linear relationships
# correlation coefficient - how similar is the pairwise movement of two variables around their averages
# between -1 and 1
# the closer to 1 or -1, the more linear the relationship

data = pd.read_csv('assets.csv', parse_dates=['date'], index_col='date')
data = data.dropna().info()
daily_returns = data.pct_change()
sns.jointplot(x='sp500', y='nasdaq', data=daily_returns)

correlations = returns.corr()
sns.heatmap(correlations, annot=True)

# Exercises
# Inspect data here
print(data.info())

# Calculate year-end prices here
annual_prices = data.resample('A').last()

# Calculate annual returns here
annual_returns = annual_prices.pct_change()

# Calculate and print the correlation matrix here
correlations = annual_returns.corr()
print(correlations)

# Visualize the correlations as heatmap here
sns.heatmap(correlations, annot=True)
plt.show()

# Manipulating time series data
# Value weighted index
# share price * number of shares => market value
# Larger companies have a larger weight and will have a larger impact on index

nyse = pd.read_csv('listings.xlsx', sheet_name='nyse', na_values='n/a')
nyse.set_index('Stock Symbol', inplace=True)
nyse.dropna(subset=['Sector'], inplace=True)
nyse['Market Capitalization'] /= 1e6
components = nyse.groupby(['Sector'])['Market Capitalization'].nlargest(1)
components.sort_values(ascending=False)
tickers = components.index.get_level_values('Stock Symbol')
tickers.tolist()

columns = ['Company Name', 'Market Capitalization', 'Last Sale']
component_info = nyse.loc[tickers, columns]
pd.options.display.float_format = '{:,.2f}'.format

data = pd.read_csv('stocks.csv', parse_dates=["Date"], index_col='Date').loc[:, tickers.tolist()]

# Exercises
# Inspect listings
print(listings.info())

# Move 'stock symbol' into the index
listings.set_index('Stock Symbol', inplace=True)

# Drop rows with missing 'sector' data
listings.dropna(subset=['Sector'], inplace=True)

# Select companies with IPO Year before 2019
listings = listings[listings['IPO Year'] < 2019]

# Inspect the new listings data
print(listings.info())

# Show the number of companies per sector
print(listings.groupby('Sector').size().sort_values(ascending=False))

# Select largest company for each sector
components = listings.groupby(['Sector'])['Market Capitalization'].nlargest(1)

# Print components, sorted by market cap
print(components.sort_values(ascending=False))

# Select stock symbols and print the result
tickers = components.index.get_level_values('Stock Symbol')
print(tickers)

# Print company name, market cap, and last price for each component
info_cols = ['Company Name', 'Market Capitalization', 'Last Sale']
print(listings.loc[tickers, info_cols].sort_values('Market Capitalization', ascending=False))

# Print tickers
print(tickers)

# Import prices and inspect result
stock_prices = pd.read_csv('stock_prices.csv', parse_dates=['Date'], index_col='Date')
print(stock_prices.info())

# Calculate the returns
price_return = stock_prices.iloc[-1].div(stock_prices.iloc[0]).sub(1).mul(100)

# Plot horizontal bar chart of sorted price_return
price_return.sort_values().plot(kind='barh', title='Stock Price Returns')
plt.show()

# number of shares
shares = components['Market Capitalization'].div(components['Last Sale'])

data = pd.read_csv('stocks.csv', parse_dates=['Date'], index_col='Date').loc[:, tickers.tolist()]
market_cap_series = data.mul(no_shares)
market_cap_series.first('D').append(market_cap_series.last('D'))

agg_mcap = market_cap_series.sum(axis=1)  #total market cap
agg_mcap(title='Aggregate Market Cap')

index = agg_mcap.div(agg_mcap.iloc[0]).mul(100) #creates value based index
index.plot(title='Market-Cap Weighted Index')

# Inspect listings and print tickers
print(listings.info())
print(tickers)

# Select components and relevant columns from listings
components = listings.loc[tickers, ['Market Capitalization', 'Last Sale']]

# Print the first rows of components
print(components.head())

# Calculate the number of shares here
no_shares = components['Market Capitalization'].div(components['Last Sale'])

# Print the sorted no_shares
print(no_shares.sort_values(ascending=False))

# Select the number of shares
no_shares = components['Number of Shares']
print(no_shares.sort_values())

# Create the series of market cap per ticker
market_cap = stock_prices.mul(no_shares)

# Select first and last market cap here
first_value = market_cap.iloc[0]
last_value = market_cap.iloc[-1]

# Concatenate and plot first and last market cap here
pd.concat([first_value, last_value], axis=1).plot(kind='barh')
plt.show()

# Aggregate and print the market cap per trading day
raw_index = market_cap_series.sum(axis=1)
print(raw_index)

# Normalize the aggregate market cap here
index = raw_index.div(raw_index.iloc[0]).mul(100)
print(index)

# Plot the index here
index.plot(title='Market-Cap Weighted Index')
plt.show()

# Evaluated index performance
# look at index return
# performance vs. benchmakr

agg_market_cap = market_cap_series.sum(axis=1)
index = agg_market_cap.div(agg_market_cap.iloc[0]).mul(100)
index.plot(title='Market-Cap Weighted Index')

agg_market_cap.ilocc[-1] - agg_market_cap.iloc[0]

# Value contribution by stock
change = market_cap_series.first('D').append(market_cap_series.last('D'))
change.diff().iloc[-1].sort_values()

market_cap = components['Market Capitalization']
weights = market_cap.div(market_cap.sum())
weights.sort_values().mul(100)
# Value weighted component returns
index_return = (index.iloc[-1] / index.iloc[0] -1) * 100
weighted_returns = weights.mul(index_return)
weighted_returns.sort_values().plot(kind='barh')

# performance vs benchmark
data = index.to_frame('Index')
data['SP500'] = pd.read_csv('sp500.csv', parse_dates=['Date'], index_col='Date')
data.SP500 = data.SP500.div(data.SP500.iloc[0], axis=0).mul(100)

def multi_period_return(r):
    return (np.prod(r +1) -1) * 100
data.pct_change().rolling('30D').apply(multi_period_return).plot()

# Calculate and print the index return here
index_return = (index.iloc[-1] / index.iloc[0] -1) * 100
print(index_return)

# Select the market capitalization
market_cap = components['Market Capitalization']

# Calculate the total market cap
total_market_cap = market_cap.sum()

# Calculate the component weights, and print the result
weights = market_cap.div(total_market_cap)
print(weights.sort_values())

# Calculate and plot the contribution by component
weights.mul(index_return).sort_values().plot(kind='barh')
plt.show()

# Calculate and print the index return here
index_return = (index.iloc[-1] / index.iloc[0] -1) * 100
print(index_return)

# Select the market capitalization
market_cap = components['Market Capitalization']

# Calculate the total market cap
total_market_cap = market_cap.sum()

# Calculate the component weights, and print the result
weights = market_cap.div(total_market_cap)
print(weights.sort_values())

# Calculate and plot the contribution by component
weights.mul(index_return).sort_values().plot(kind='barh')
plt.show()

# Inspect data
print(data.info())
print(data.head())

# Create multi_period_return function here
def multi_period_return(r):
    return (np.prod(r + 1) -1) * 100

# Calculate rolling_return_360
rolling_return_360 = data.pct_change().rolling('360D').apply(multi_period_return)

# Plot rolling_return_360 here
rolling_return_360.plot(title='Rolling 360D Return')
plt.show()

# Index Correlation
data = DataReader(tickers, 'google', start='2016', end='2017')['Close']
data.info()

daily_returns = data.pct_change()
correlations = daily_returns.corr()
sns.heatmap(correlations, annot=True)
plt.xticks(rotation=45)
plt.title('Daily Return Correlations')

# Saving to Excel
correlations.to_excel(excel_writer = 'correlations.xls',
    sheet_name='corrrelations',
    startrow=1,
    startcol=1)

# Saving to multiple worksheets
data.index = data.index.date
with pd.Excel_writer('stock_data.xlsx') as writer:
    corr.to_excel(excel_writer=writer, sheet_name='correlations')
    data.to_excel(excel_writer=writer, sheet_name='prices')
    data.pct_change.to_excel(excel_writer=writer, sheet_name='returns')

# Exercises
# Inspect stock_prices here
print(stock_prices.info())

# Calculate the daily returns
returns = stock_prices.pct_change()

# Calculate and print the pairwise correlations
correlations = returns.corr()
print(correlations)

# Plot a heatmap of daily return correlations
sns.heatmap(correlations, annot=True)
plt.title('Daily Returns Correlations')
plt.show()

# Inspect index and stock_prices
print(index.info())
print(stock_prices.info())

# Join index to stock_prices, and inspect the result
data = stock_prices.join(index)
print(data.info())

# Create index & stock price returns
returns = data.pct_change()

# Export data and data as returns to excel
with pd.ExcelWriter('data.xls') as writer:
    data.to_excel(excel_writer=writer, sheet_name='data')
    returns.to_excel(excel_writer=writer, sheet_name='returns')


# -*- coding: utf-8 -*-

import os
from setuptools import find_packages, setup

with open(os.path.join(os.path.dirname(__file__), 'README.md')) as readme:
    README = readme.read()

# allow setup.py to be run from any path
os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir)))

with open("botshot/version.py") as fp:
    version_dict = {}
    exec(fp.read(), version_dict)
    __version__ = version_dict['__version__']

setup(
    name='botshot',
    version=__version__,
    packages=find_packages(),
    include_package_data=True,
    license='GPL',
    description='A framework for creating stateful chatbots on Django.',
    long_description=README,
    long_description_content_type='text/markdown',
    url='https://github.com/prihoda/botshot',
    author='David Příhoda, Matúš Žilinec, Jakub Drdák',
    author_email='david.prihoda@gmail.com, zilinec.m@gmail.com',
    entry_points={
          'console_scripts': [
              'bots = botshot.bots:main'
          ]
      },
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Web Environment',
        'Framework :: Django',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        # 'Programming Language :: Python :: 3.7',  Not compatible yet, waiting for Celery 4.3 !
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
        'Topic :: Communications :: Chat',
    ],
    install_requires=['django>=2.1.5', 'networkx', 'requests', 'six', 'sqlparse', 'wit==4.3.0', 'wheel', 'redis', 'Pillow', 'jsonfield',
                      'pytz', 'unidecode', 'emoji', 'elasticsearch', 'celery>=4.1.1', 'python-dateutil', 'pyyaml', 'djangorestframework',
                      'pytest', 'pytest-django', 'mock'],
)


#!/usr/bin/env python3

import sys

template = """\"\"\"Day {day}: {description}\"\"\"

from typing import List
from util import text_input, file_input, assert2

example_input = \"\"\"
{example_input}
\"\"\"

def day{day}_1(lines: List[str]) -> int:
    return 0

assert2(0, day{day}_1(text_input(example_input, str)))
assert2(0, day{day}_1(file_input({day}, str)))

def day{day}_2(lines: List[str]) -> int:
    return 0

assert2(0, day{day}_2(text_input(example_input, str)))
assert2(0, day{day}_2(file_input({day}, str)))
"""

def fail(msg: str) -> None:
    print(msg, file=sys.stderr)
    sys.exit(1)

def main() -> None:
    if len(sys.argv) < 3:
        fail(sys.argv[0] + ": <day> <description>")
    day = int(sys.argv[1])
    desc = sys.argv[2]
    filepath = "day{0:02d}.py".format(day)
    example_input = sys.stdin.read().strip()
    with open(filepath, "x") as f:
        content = template.format(
            day=day, description=desc, example_input=example_input
        )
        f.write(content)
    print("wrote", filepath, file=sys.stderr)

if __name__ == "__main__":
    main()


import numpy as np
import time

def problem_generator(n: int, ave_size: int):
	# sizes is how much space an objects takes up in the knapsack
	sizes = np.random.randint(1, ave_size * 2, n + 1, dtype=int)
	# values shows how much each objects is worth in $$ google-bucks $$
	values = np.random.randint(1, ave_size * 2, n + 1)*np.random.random(n+1)
	# making index 0 of both arrays useless
	values[0] = 0.0
	sizes[0] = 0
	return sizes, values

def knapsack_bool(i, size):
	# if the size of the knapsack has been filled
	if size == 0:
		return True
	# if the knapsack cannot be filled exactly
	if size < 0:
		return False
	# if the available stones to use has run out
	if i <= 0:
		return False
	# try skipping/ reducing the size of the knapsack with the next stone
	return knapsack_bool(i - 1, size - S[i]) or knapsack_bool(i - 1, size)

def optimize_knaps(i, k1, k2)->float:
	if k1 < 0 or k2 < 0:
		return -1000000000000.0
	if i <= 0:
		return 0.0

	return max(optimize_knaps(i - 1, k1 - S[i], k2) + V[i], optimize_knaps(i - 1, k1, k2 - S[i]) + V[i],
			   optimize_knaps(i - 1, k1, k2))

def optimize_memo_knaps(i, k1, k2)->float:
	if k1 < 0 or k2 < 0:
		return -1000000000000.0
	if i <= 0:
		return 0.0

	# if answer is already calculated, return that
	if Done[i][k1][k2]:
		return C[i][k1][k2]

	else:

		# solve a unique problem then return it after saving
		C[i][k1][k2] = max(optimize_memo_knaps(i - 1, k1 - S[i], k2) + V[i],
		                   optimize_memo_knaps(i - 1, k1, k2 - S[i]) + V[i],
		                   optimize_memo_knaps(i - 1, k1, k2))
		Done[i][k1][k2] = True
		return C[i][k1][k2]

def optimize_dp_knaps(i, k1, k2)->float:
	for n in range(1, i+1):
		for m in range(k1+1):
			for l in range(k2+1):

				# catches indexes less than zero
				if m - S[n] < 0:
					add_one = -1000000000
				else:
					add_one = C[n - 1][m - S[n]][l] + V[n]
				if l - S[n] < 0:
					add_two = -1000000000
				else:
					add_two = C[n - 1][m][l - S[n]] + V[n]

				add_none = C[n - 1][m][l]

				C[n][m][l] = max(add_one, add_two, add_none)

	return C[i][k1][k2]

# TODO Write a dp for the simple knapsack problem

def optimize_knapsacks_cache(n, k1, k2):
	return np.zeros([n + 1, k1+1, k2+1], dtype=float)

def run_simple():
	global N
	global K
	global S
	global V
	N = valid_int('How many objects are there to choose from?')
	K = valid_int('How big is the knapsack? I suggest between 10 and 200.')
	ave_size = valid_int(f"Average size of the stones? I suggest around {int(K1/5)}")
	S, V = problem_generator(N, ave_size)
	print(f"Set S if {S}. Size of sack is {K}")
	return knapsack_bool(N, K1)

def run_double(func):
	global N
	global K1
	global K2
	global S
	global V
	global C
	global Done
	N = valid_int('How many objects are there to choose from?')
	K1 = valid_int('How big is knapsack 1? I suggest between 10 and 200.')
	K2 = valid_int('How big is knapsack 2? I suggest between 10 and 200.')
	ave_size = valid_int(f"Average size of the stones? I suggest between {K1/2} and {K2/2}")
	S, V = problem_generator(N, ave_size)
	Done = np.full([N + 1, K1 + 1, K2 + 1], False)
	print(f"Set S if {S}. Values inside the sack is {V}")
	print(f"Best value able to be carried with knapsacks of size {K1} and {K2} is...")
	C = optimize_knapsacks_cache(N, K1, K2)
	return func(N, K1, K2)

def run_memo_dp_comparision():
	global N
	global K1
	global K2
	global Done
	N = valid_int('How many objects are there to choose from?')
	K1 = valid_int('How big is knapsack 1? I suggest between 10 and 200.')
	K2 = valid_int('How big is knapsack 2? I suggest between 10 and 200.')
	average_sizes = np.random.randint(5, 120, 10)
	with open('trials.txt', 'a') as file:
		file.write(f"N={N},K1={K1},K2={K2}\naverage Sizes={average_sizes}\n\n\n")

	for ave_size in average_sizes:
		i = 1
		total_dp_time = 0.0
		total_memo_time = 0.0
		with open('trials.txt', 'a') as file:
			file.write("-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-\n")
			file.write(f"starting new average size of {ave_size}\n\n Trial\t | memoizing time\t "
			           f"| dynamic program time\t | answers match\n")

		for trial in range(20):
			global S
			global V
			global C
			S, V = problem_generator(N, ave_size)
			Done = np.full([N + 1, K1 + 1, K2 + 1], False)
			C = optimize_knapsacks_cache(N, K1, K2)

			# test memoizing
			start = time.time()
			memo = optimize_memo_knaps(N, K1, K2)
			mid = time.time()

			# reset cache
			C = optimize_knapsacks_cache(N, K1, K2)

			# test dynamic program
			mid2 = time.time()
			dp = optimize_dp_knaps(N, K1, K2)
			end = time.time()

			# to keep track of average times
			total_dp_time += end - mid2
			total_memo_time += mid - start

			with open('trials.txt', 'a') as file:
				file.write(f"{trial+1}:\t  {mid - start}\t | {end-mid2}\t | {memo==dp} {memo}  {dp}\n")
			i += 1

		with open('trials.txt', 'a') as file:
			file.write(f'\nAverage time for ave_size= {ave_size}. Memo= {total_memo_time/i}. DP = {total_dp_time/i}.\n')

def valid_int(message):
	print(message)
	try:
		response = int(input('-->\t'))
	except ValueError:
		print("Enter a valid integer.")
		return valid_int(message)
	return response

# i didn't want to deal with setting up a class, so i broke the golden CS rule!
# declaring global variables to random stuff
N = 5
K1 = 100
K2 = 100
K = 100
S = [1, 2]
V = [1.1, 1.2]
C = [[1.2], [1.1]]
Done = [True]


# A utility class that represents an individual node in a BST
class Node:
	def __init__(self,key):
		self.left = None
		self.right = None
		self.val = key

# a utility function to insert a new node with the given key value
def insertnode(root,node):
	if root is None:
		root = node
	else:
		if root.val > node.val:
			if root.left is None:
				root.left = node
			else:
				insertnode(root.left,node)
		else:
			if root.right is None:
				root.right = node
			else:
				insertnode(root.right, node)

r = Node(50)
insertnode(r,Node(30))
insertnode(r,Node(20))


from __future__ import absolute_import

'''Resnet for cifar dataset.
Ported form
https://github.com/facebook/fb.resnet.torch
and
https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py
(c) YANG, Wei
'''
import torch.nn as nn
import math

__all__ = ['resnet']

def conv3x3(in_planes, out_planes, stride=1):
    "3x3 convolution with padding"
    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
                     padding=1, bias=False)

class BasicBlock(nn.Module):
    expansion = 1
    # Basicblock for cifar10 / cifar100
    def __init__(self, inplanes, planes, stride=1, downsample=None):
        super(BasicBlock, self).__init__()
        self.conv1 = conv3x3(inplanes, planes, stride)
        self.bn1 = nn.BatchNorm2d(planes)
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = conv3x3(planes, planes)
        self.bn2 = nn.BatchNorm2d(planes)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out += residual
        out = self.relu(out)

        return out

class Bottleneck(nn.Module):
    expansion = 4

    def __init__(self, inplanes, planes, stride=1, downsample=None):
        super(Bottleneck, self).__init__()
        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
                               padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(planes * 4)
        self.relu = nn.ReLU(inplace=True)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out += residual
        out = self.relu(out)

        return out

class ResNet(nn.Module):
   # Basicblock for cifar10 / cifar100
    def __init__(self, depth, num_classes=1000, block_name='BasicBlock'):
        super(ResNet, self).__init__()
        # Model type specifies number of layers for CIFAR-10 model    以110层为例
        if block_name.lower() == 'basicblock':
            assert (depth - 2) % 6 == 0, 'When use basicblock, depth should be 6n+2, e.g. 20, 32, 44, 56, 110, 1202'
            n = (depth - 2) // 6   # 18
            block = BasicBlock
        elif block_name.lower() == 'bottleneck':                                                    #     12
            assert (depth - 2) % 9 == 0, 'When use bottleneck, depth should be 9n+2, e.g. 20, 29, 47, 56, 110, 1199'
            n = (depth - 2) // 9   # 12
            block = Bottleneck
        else:
            raise ValueError('block_name shoule be Basicblock or Bottleneck')

        self.inplanes = 16
        self.conv1 = nn.Conv2d(3, 16, kernel_size=3, padding=1, bias=False)  # 3*3 通道 3 --> 16
        self.bn1 = nn.BatchNorm2d(16)                                            # 通道 16 --> 16
        self.relu = nn.ReLU(inplace=True)                                        # 通道 16 --> 16

        self.layer1 = self._make_layer(block, 16, n)                             # 通道 16 --> 16
        self.layer2 = self._make_layer(block, 32, n, stride=2)                   # 通道 16 --> 32
        self.layer3 = self._make_layer(block, 64, n, stride=2)                   # 通道 32 --> 64

        self.avgpool = nn.AvgPool2d(8)  # 池化窗口 大小为8*8
        self.fc = nn.Linear(64 * block.expansion, num_classes)                  # 64 --> 100

        # self.convShare = nn.Conv2d(3, 16, kernel_size=3, padding=1, bias=False)  # 共享投影层
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
                    #     什么块  输出通道 多少块    步长
    def _make_layer(self, block, planes, blocks, stride=1):   # 通用块
        downsample = None

        #     步幅 不为1      上一个层级输出通道  不等于 这一层级输出通道
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(planes * block.expansion),
            )

        layers = []
        #               输入通道 = 上一级大Blocks块的输出通道
        #                       输入通道   输出通道
        layers.append(block(self.inplanes, planes, stride, downsample))  # 只有刚进该大块的时候 把图片尺寸卷成了一半
        self.inplanes = planes * block.expansion    # 输入通道等于第一个小Block块的输出通道
        for i in range(1, blocks):  # 每个大块 各有18个 BasicBlock  填入剩下的17个小Block块
                            #     输入通道        输出通道
            layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)    # 32x32

        x = self.layer1(x)  # 32x32
        x = self.layer2(x)  # 16x16
        x = self.layer3(x)  # 8x8

        x = self.avgpool(x)
        x = x.view(x.size(0), -1)
        x = self.fc(x)

        return x

def resnet(**kwargs):
    """
    Constructs a ResNet model.
    """
    return ResNet(**kwargs)


class Student:
    def __init__(self, name, surname, gender):
        self.name = name
        self.surname = surname
        self.gender = gender
        self.finished_courses = []
        self.courses_in_progress = []
        self.grades = {}

    def rate_hw(self, lecturer, course, grade):
        if isinstance(lecturer, Lecturer) and course in self.courses_in_progress and course in lecturer.courses_attached:
            if course in lecturer.grades:
                lecturer.grades[course] += [grade]
            else:
                lecturer.grades[course] = [grade]
        else:
            return 'Ошибка'

    def _not_genius_(self):
        averages = {}
        for key, values in self.grades.items():
            average = sum(values) / len(values)
            averages = average
        return averages

    def __str__(self) :
        return  f"""
Имя: {self.name}
Фамилия: {self.surname}
Средняя оценка за домашние задания: {Student._not_genius_(self)}
Курсы в процессе изучения: {", ".join(self.courses_in_progress)}
Завершенные курсы: {", ".join(self.finished_courses)}
                """

    def __lt__(self, other):
        a = self._not_genius_()
        b = other._not_genius_()
        print(a < b)

class Mentor:
    def __init__(self, name, surname):
        self.name = name
        self.surname = surname
        self.courses_attached = []

class Lecturer(Mentor):
    def __init__(self, name, surname):
        super().__init__(name, surname)
        self.courses_in_progress = []
        self.grades = {}

    def _not_genius_(self):
        averages = {}
        for key, values in self.grades.items():
            average = sum(values) / len(values)
            averages = average
        return averages

    def __str__(self) :
        return  f"""
Имя: {self.name}
Фамилия: {self.surname}
Средняя оценка за лекции: {(Lecturer._not_genius_(self))}
                """
    def __lt__(self, other):
        a = self._not_genius_()
        b = other._not_genius_()
        print(a < b)

class Reviewer(Mentor):
    def rate_hw(self, student, course, grade):
        if isinstance(student, Student) and course in student.courses_in_progress:
            if course in student.grades:
                student.grades[course] += [grade]
            else:
                student.grades[course] = [grade]
        else:
            return 'Ошибка'

    def __str__(self):
        return f"""
Имя: {self.name}
Фамилия: {self.surname}
                """

def student_midle_rate(student_list, course):
    total_grades = []
    count = 0
    for student in student_list:
        if course in student.courses_in_progress and course in student.grades:
            grades = student.grades[course]
            total_grades.extend(grades)
            count += len(grades)
    if count > 0:
        average_grade = sum(total_grades) / count
        return round(average_grade, 1)
    else:
        return 'Нет оценок для данного курса'

eren = Student('Eren', 'Eger', 'male')
armin = Student('Armin', 'Alert', 'male')
mikasa = Student('Mikasa', 'Ackerman', 'female')

eren.courses_in_progress += ['Python',]
armin.courses_in_progress += ['Python', 'Java', 'SQL']
mikasa.courses_in_progress += ['C++', 'C#']
eren.finished_courses += ['Ведение в програмирование']
armin.finished_courses += ['Ведение в програмирование']
mikasa.finished_courses += ['Ведение в програмирование']

erwin = Lecturer('Erwin', 'Smith')
hanji = Lecturer('Hanji', 'Zoe')

erwin.courses_attached += ['Python', 'Java']
hanji.courses_attached += ['C++', 'C#', 'SQL']

grisha = Reviewer('Grisha', 'Eger')

eren.rate_hw(erwin, 'Python', 10)
armin.rate_hw(erwin, 'Java', 8)
armin.rate_hw(erwin, 'Python', 5)
mikasa.rate_hw(hanji, 'C++', 9)
armin.rate_hw(hanji, 'SQL', 10)
mikasa.rate_hw(hanji, 'C#', 3)

grisha.rate_hw(eren, 'Python', 6)
grisha.rate_hw(armin, 'Java', 10)
grisha.rate_hw(armin, 'Python', 9)
grisha.rate_hw(armin, 'SQL', 9)
grisha.rate_hw(mikasa, 'C++', 8)
grisha.rate_hw(mikasa, 'C#', 6)

# print(grisha)
# print(erwin)
# print(eren)
# eren > mikasa
# hanji > erwin

students_list = [eren, mikasa, armin]

middle = student_midle_rate(students_list, 'Python')
print(middle)


import abc
import enum
import types

import dbus

from into_dbus_python import xformers

_IDENTITY = lambda x: x # pragma: no cover

def _option_to_tuple(value, default):
    """
    Converts a Python "option" type, None or a value, to a tuple.

    :param object value: any value
    :param object default: a valid default to use if value is None
    :returns: a tuple encoding the meaning of value
    :rtypes: bool * object
    """
    return (False, default) if value is None else (True, value)

def _info_to_xformer(specs):
    """
    Function that yields a xformer function.

    :param specs: specifications for this function
    :type names: iterable of triples, name, inxform function, sig
    :return: a transformation function
    :rtype: (list of object) -> (list of object)
    """
    inxforms = [y for (_, y, _) in specs]
    outxforms = [f for (f, _) in xformers("".join(z for (_, _, z) in specs))]
    expected_length = len(specs)

    def xformer(objects):
        """
        Transforms a list of objects to dbus-python types.

        :param objects: list of objects
        :type objects: list of object
        :returns: a transformed list of objects
        :rtype: list of object
        """
        if len(objects) != expected_length:
            raise ValueError("wrong number of objects")

        return \
           [x for (x, _) in (f(g(a)) for \
           ((f, g), a) in zip(zip(outxforms, inxforms), objects))]

    return xformer

def _xformers(key_to_sig):
    """
    Get a map from keys to functions from a map of names to signatures.

    :param key_to_sig: a map from keys to signatures
    :type key_to_sig: dict of object * str
    :returns: a map from keys to functions
    :rtype: dict of object * xformation function
    """
    return dict(
       (method, ([n for (n, _, _) in specs], _info_to_xformer(specs))) for \
       (method, specs) in key_to_sig.items())

class InterfaceSpec(abc.ABC):
    """
    Parent class for an interface hierarchy.
    """
    # pylint: disable=too-few-public-methods

    INTERFACE_NAME = abc.abstractproperty(doc="interface name")
    INPUT_SIGS = \
       abc.abstractproperty(doc="map from method name to input signatures")
    OUTPUT_SIGS = \
       abc.abstractproperty(doc="map from method name to output signatures")
    XFORMERS = abc.abstractproperty(doc="map from method name to xformer")
    PROPERTY_SIGS = abc.abstractproperty(doc="map from property names to sigs")

class ObjectManagerSpec(InterfaceSpec):
    """
    org.freedesktop.DBus.ObjectManager interface
    """
    # pylint: disable=too-few-public-methods

    class MethodNames(enum.Enum):
        """
        Method names.
        """
        GetManagedObjects = "GetManagedObjects"

    class PropertyNames(enum.Enum):
        """
        Property names.
        """
        pass

    INTERFACE_NAME = "org.freedesktop.DBus.ObjectManager"

    INPUT_SIGS = {
       MethodNames.GetManagedObjects: (),
    }

    OUTPUT_SIGS = {
       MethodNames.GetManagedObjects: "a{oa{sa{sv}}}"
    }
    XFORMERS = _xformers(INPUT_SIGS)

    PROPERTY_SIGS = {}

class FilesystemSpec(InterfaceSpec):
    """
    Filesystem interface.
    """
    # pylint: disable=too-few-public-methods

    class MethodNames(enum.Enum):
        """
        Names of the methods of the Filesystem class.
        """
        SetName = "SetName"

    class PropertyNames(enum.Enum):
        """
        Names of the properties of the Filesystem interface.
        """
        Name = "Name"
        Pool = "Pool"
        Uuid = "Uuid"

    INTERFACE_NAME = 'org.storage.stratis1.filesystem'

    INPUT_SIGS = {
       MethodNames.SetName: (("name", _IDENTITY, "s"),),
    }
    OUTPUT_SIGS = {
       MethodNames.SetName: "bqs",
    }
    XFORMERS = _xformers(INPUT_SIGS)

    PROPERTY_SIGS = {
       PropertyNames.Name: "s",
       PropertyNames.Pool: "o",
       PropertyNames.Uuid: "s",
    }

class ManagerSpec(InterfaceSpec):
    """
    Manager interface.
    """
    # pylint: disable=too-few-public-methods

    class MethodNames(enum.Enum):
        """
        The method names of the manager interface.
        """
        ConfigureSimulator = "ConfigureSimulator"
        CreatePool = "CreatePool"
        DestroyPool = "DestroyPool"

    class PropertyNames(enum.Enum):
        """
        Names of the properties of the manager interface.
        """
        ErrorValues = "ErrorValues"
        RedundancyValues = "RedundancyValues"
        Version = "Version"

    INTERFACE_NAME = 'org.storage.stratis1.Manager'

    INPUT_SIGS = { # pragma: no cover
        MethodNames.ConfigureSimulator : (("denominator", _IDENTITY, "u"),),
        MethodNames.CreatePool :
           (
              ("name", _IDENTITY, "s"),
              ("redundancy", (lambda x: _option_to_tuple(x, 0)), "(bq)"),
              ("force", _IDENTITY, "b"),
              ("devices", _IDENTITY, "as"),
           ),
        MethodNames.DestroyPool : (("pool", _IDENTITY, "o"),),
    }
    OUTPUT_SIGS = {
        MethodNames.ConfigureSimulator : "qs",
        MethodNames.CreatePool : "(oas)qs",
        MethodNames.DestroyPool : "bqs",
    }
    XFORMERS = _xformers(INPUT_SIGS)

    PROPERTY_SIGS = {
       PropertyNames.ErrorValues: "a(sq)",
       PropertyNames.RedundancyValues: "a(sq)",
       PropertyNames.Version: "s"
    }

class PoolSpec(InterfaceSpec):
    """
    Pool interface.
    """
    # pylint: disable=too-few-public-methods

    class MethodNames(enum.Enum):
        """
        Names of the methods of the Pool class.
        """
        AddDevs = "AddDevs"
        CreateFilesystems = "CreateFilesystems"
        DestroyFilesystems = "DestroyFilesystems"
        SetName = "SetName"

    class PropertyNames(enum.Enum):
        """
        Names of the properties of the manager interface.
        """
        Name = "Name"
        Uuid = "Uuid"

    INTERFACE_NAME = 'org.storage.stratis1.pool'

    INPUT_SIGS = { # pragma: no cover
       MethodNames.AddDevs:
          (("force", _IDENTITY, "b"), ("devices", _IDENTITY, "as"),),
       MethodNames.CreateFilesystems: (("specs", _IDENTITY, "as"),),
       MethodNames.DestroyFilesystems: (("filesystems", _IDENTITY, "ao"),),
       MethodNames.SetName: (("name", _IDENTITY, "s"),)
    }
    OUTPUT_SIGS = {
       MethodNames.AddDevs: "asqs",
       MethodNames.CreateFilesystems: "a(os)qs",
       MethodNames.DestroyFilesystems: "asqs",
       MethodNames.SetName: "bqs"
    }
    XFORMERS = _xformers(INPUT_SIGS)

    PROPERTY_SIGS = {
       PropertyNames.Name: "s",
       PropertyNames.Uuid: "s",
    }

def _prop_builder(spec):
    """
    Returns a function that builds a property interface based on 'spec'.

    :param spec: the interface specification
    :type spec: type, a subtype of InterfaceSpec
    """

    def builder(namespace):
        """
        The property class's namespace.

        :param namespace: the class's namespace
        """

        def build_property(prop): # pragma: no cover
            """
            Build a single property getter for this class.

            :param prop: the property
            """

            def dbus_func(proxy_object): # pragma: no cover
                """
                The property getter.
                """
                return proxy_object.Get(
                   spec.INTERFACE_NAME,
                   prop.name,
                   dbus_interface=dbus.PROPERTIES_IFACE
                )

            return dbus_func

        for prop in spec.PropertyNames:
            namespace[prop.name] = staticmethod(build_property(prop)) # pragma: no cover

    return builder

def _iface_builder(spec):
    """
    Returns a function that builds a method interface based on 'spec'.

    :param spec: the interface specification
    :type spec: type, a subtype of InterfaceSpec
    """

    def builder(namespace):
        """
        Builds the class.

        :param namespace: the class's namespace
        """

        def build_method(method):
            """
            Build a single method for this class.

            :param method: the method
            """
            (names, func) = spec.XFORMERS[method]

            def dbus_func(proxy_object, **kwargs): # pragma: no cover
                """
                The function method spec.
                """
                if frozenset(names) != frozenset(kwargs.keys()):
                    raise ValueError("Bad keys")
                args = \
                   [v for (k, v) in \
                   sorted(kwargs.items(), key=lambda x: names.index(x[0]))]
                xformed_args = func(args)
                dbus_method = getattr(proxy_object, method.name)
                return dbus_method(
                   *xformed_args,
                   dbus_interface=spec.INTERFACE_NAME
                )

            return dbus_func

        for method in spec.MethodNames:
            namespace[method.name] = staticmethod(build_method(method))

        namespace['Properties'] = \
           types.new_class(
              "Properties",
              bases=(object,),
              exec_body=_prop_builder(spec)
           )

    return builder

ObjectManager = types.new_class(
   "ObjectManager",
   bases=(object,),
   exec_body=_iface_builder(ObjectManagerSpec)
)
Filesystem = types.new_class(
   "Filesystem",
   bases=(object,),
   exec_body=_iface_builder(FilesystemSpec)
)
Manager = types.new_class(
   "Manager",
   bases=(object,),
   exec_body=_iface_builder(ManagerSpec)
)
Pool = \
   types.new_class("Pool", bases=(object,), exec_body=_iface_builder(PoolSpec))


import datetime
import sqlalchemy as sa
import psycopg2

from . import DatabaseConnectionError, SQLError

from .models import InfernalJob, JOB_CHUNK_STATUS_CHOICES

async def save_infernal_job(engine, job_id, priority):
    """
    Create infernal job
    :param engine: params to connect to the db
    :param job_id: id of the job
    :param priority: priority of the job, high or low
    """
    try:
        async with engine.acquire() as connection:
            try:
                await connection.scalar(
                    InfernalJob.insert().values(
                        job_id=job_id,
                        submitted=datetime.datetime.now(),
                        priority=priority,
                        status=JOB_CHUNK_STATUS_CHOICES.pending)
                )
            except Exception as e:
                raise SQLError("Failed to save_infernal_job for job_id = %s" % job_id) from e
    except psycopg2.Error as e:
        raise DatabaseConnectionError("Failed to open database connection in save_infernal_job for "
                                      "job_id = %s" % job_id) from e

async def set_infernal_job_status(engine, job_id, status):
    """
    Update the status of the infernal job
    :param engine: params to connect to the db
    :param job_id: id of the job
    :param status: an option from consumer.JOB_CHUNK_STATUS
    :return: None
    """
    finished = None
    if status == JOB_CHUNK_STATUS_CHOICES.success or \
       status == JOB_CHUNK_STATUS_CHOICES.error or \
       status == JOB_CHUNK_STATUS_CHOICES.timeout:
        finished = datetime.datetime.now()

    submitted = None
    if status == JOB_CHUNK_STATUS_CHOICES.started:
        submitted = datetime.datetime.now()

    try:
        async with engine.acquire() as connection:
            try:
                if submitted:
                    query = sa.text('''
                        UPDATE infernal_job
                        SET status = :status, submitted = :submitted
                        WHERE job_id = :job_id
                        RETURNING *;
                    ''')

                    infernal_job = None  # if connection didn't return any rows, return None
                    async for row in connection.execute(query, job_id=job_id, status=status, submitted=submitted):
                        infernal_job = row.id
                    return infernal_job
                elif finished:
                    query = sa.text('''
                        UPDATE infernal_job
                        SET status = :status, finished = :finished
                        WHERE job_id = :job_id
                        RETURNING *;
                    ''')

                    infernal_job = None  # if connection didn't return any rows, return None
                    async for row in connection.execute(query, job_id=job_id, status=status, finished=finished):
                        infernal_job = row.id
                    return infernal_job
                else:
                    query = sa.text('''
                        UPDATE infernal_job
                        SET status = :status
                        WHERE job_id = :job_id
                        RETURNING *;
                    ''')

                    infernal_job = None  # if connection didn't return any rows, return None
                    async for row in connection.execute(query, job_id=job_id, status=status):
                        infernal_job = row.id
                    return infernal_job
            except Exception as e:
                raise SQLError("Failed to set_job_chunk_status in the database,"
                               " job_id = %s, status = %s" % (job_id, status)) from e
    except psycopg2.Error as e:
        raise DatabaseConnectionError("Failed to open connection to the database in set_job_chunk_status, "
                                      "job_id = %s" % job_id) from e

async def set_consumer_to_infernal_job(engine, job_id, consumer_ip):
    """
    Update the infernal_job table to register the consumer who will run the job
    :param engine: params to connect to the db
    :param job_id: id of the job
    :param consumer_ip: ip address of the consumer
    :return: id or none
    """
    try:
        async with engine.acquire() as connection:
            try:
                query = sa.text('''
                    UPDATE infernal_job
                    SET consumer = :consumer_ip
                    WHERE job_id=:job_id
                    RETURNING *;
                ''')
                infernal_job = None  # if connection didn't return any rows, return None
                async for row in connection.execute(query, job_id=job_id, consumer_ip=consumer_ip):
                    infernal_job = row.id
                return infernal_job
            except Exception as e:
                raise SQLError("Failed to set_consumer_to_infernal_job in the database, job_id = %s" % job_id) from e
    except psycopg2.Error as e:
        raise DatabaseConnectionError("Failed to open connection to the database in set_consumer_to_infernal_job, "
                                      "job_id = %s" % job_id) from e


#%%
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import numpy as np
from xshooter_synthmag import Filter, synthcolor
from astropy.io import fits
from astroquery.vizier import Vizier
Vizier.ROW_LIMIT = 50000
plt.rcParams.update({'font.size': 22})
#%%

#%%
#directory of where the spectra are
DATA_DIR = '/datascope/subaru/data/catalogs/xshooter/fits/'

#load the filters
filt_g = Filter()
filt_g.read('/datascope/subaru/data/pfsspec/subaru/hsc/filters/fHSC-g.txt')

filt_i = Filter()
filt_i.read('/datascope/subaru/data/pfsspec/subaru/hsc/filters/fHSC-i.txt')

filt_nb515 = Filter()
filt_nb515.read('/datascope/subaru/data/pfsspec/subaru/hsc/filters/fHSC-NB515.txt')

combined = pd.read_csv('/home/cfilion1/Galaxia/ursaminor_hsc_cleaned.csv')

#%%
#attempting to get /all/ stars

import glob
#get the table that gives the stellar parameters for the stars in xshooter dr2
table = Vizier.get_catalogs('J/A+A/627/A138/tablea1')
stellar_params = table[0].to_pandas()

#get the table of stars that have comments/flags from dr2
tab2 = Vizier.get_catalogs('J/A+A/660/A34/table')
comment_flags = tab2[0].to_pandas()
#select only the stars that have flags! This table is now just a table of 'bad' stars
comment_flags = comment_flags[comment_flags.Com != ''] #finding stars with comments!

#get the set of xshooter merged and merged_scl files, these should each have extinction
#corrected spectra
merged_files = glob.glob("/datascope/subaru/data/catalogs/xshooter/fits/*merged.fits")
scl_files = glob.glob("/datascope/subaru/data/catalogs/xshooter/fits/*merged_scl.fits")
files = merged_files + scl_files

from astroquery.simbad import Simbad
customSimbad = Simbad()
#to learn what fields we can get from simbad, run below to lines:
#Simbad.get_field_description('otypes')
#Simbad.list_votable_fields()
#We want to get information on stellar types, metallicity, etc
customSimbad.add_votable_fields('otype','otypes', 'mk','fe_h','main_id')

#table from verro et al xshooter dr3 paper with Mdwarf parameters -
#this info is transcribed!
#important note - the star name from the xshooter dr3 table is different from
#the HNAME from the xshooter file, the dr3 table name should be the correct one
mdwarf_params = pd.read_csv('adtl_mdwarf_params.txt',sep='\t')
#%%
#%%
name_list = np.array([])
g_i_color = np.array([])
nb_g_color = np.array([])
fehs = np.array([])
teffs = np.array([])
loggs = np.array([])
flag = np.array([])
values = np.array([])
sp_type = np.array([])
sec_sp_type = np.array([])
var_flag = np.array([])
np_var_flag = np.array([])
mira_lpv_flag = np.array([])
#labels that lpv or mira stars have in simbad, 'micheck' is primary name and 'micheck_sec'
#is for set of secondary labels. note I'm being generous here by including candidate and '?' stars
micheck = np.array(['Mira', 'LongPeriodV*_Candidate','LongPeriodV*'])
micheck_sec = np.array(['LP?','Mi*','LP*','Mi?'])

#simbad labels for all variable type stars that are in the x-shooter spectra
mainvar_set = np.array(['PulsV*', 'Variable*', 'Mira', 'delSctV*','RSCVnV*', 'SB*',
                'LongPeriodV*_Candidate','LongPeriodV*', 'gammaDorV*', 'alf2CVnV*',
                'RRLyrae', 'RVTauV*', 'ClassicalCep','EllipVar', 'Type2Cep', 'RCrBV*'])
secvar_set = np.array(['LP?','Mi*','dS*','LP*','Mi?'])
xshoot_name = np.array([])

#loop through files
for i in range(len(files)):
    fn = os.path.join(DATA_DIR, files[i])
    hdus = fits.open(fn, memmap=False)
    xshoot_name = np.append(xshoot_name,hdus[0].header['XSL_ID'])
    # the new m-dwarf stars that are added in x-shooter dr3 all have 'M' in their x-shooter id.
    # these stars arent in the dr2 catalogue and need to be handled separately.
    if 'M' in hdus[0].header['XSL_ID']:
        # searching simbad for info on the star, using the star's name in the mdwarf dr3 table! theres a lot going on here:
        # I need to get the name for the star from the mdwarf dr3 table using the info in the xshooter spectra
        # header (hdus[0].header, remembering that for some reason the hdu HNAMES and names in the mdwarf dr3 table
        # don't match), so I am finding the star in the mdwarf dr3 table that has the same right ascension as the
        # xshooter spectra we have selected - the RA value should be identical. Finally, I search simbad using the
        # name of the star given in the mdwarf dr3 table (column 'Star').
        resm = customSimbad.query_object(mdwarf_params[(mdwarf_params.RA.str.contains(hdus[0].header['RA_FULL']))|
                        (mdwarf_params.RA==hdus[0].header['RA_FULL'])].Star.values[0])
        #add the type label that we get from simbad to the array of type names. 'OTYPE' is main name on simbad
        sp_type = np.append(sp_type, resm['OTYPE'][0])
        #add the secondary type label to our array of type names. 'OTYPES' is secondary name on simbad
        sec_sp_type = np.append(sec_sp_type, resm['OTYPES'][0])
        #check if this type is a variable using the array of names of variable types we defined above.
        # if it is variable, flag it!
        if resm['OTYPE'][0] in mainvar_set:
                var_flag = np.append(var_flag, 1)
        #if the main type is not variable, check if the secondary type is
        else:
                #check if secondary names are in the array of secondary names of variable types we defined above.
                #note 'OTYPES' is secondary name, and the 'OTYPES' is a string with (usually) a few types
                # here we are checking if any part of the 'OTYPES' for this star is in the list of secondary names
                if any(param in resm['OTYPES'][0] for param in secvar_set):
                        var_flag = np.append(var_flag, 1)
                #if niether of the main and secondary types are variable, no flag! not a variable star
                else:
                        var_flag = np.append(var_flag, 0)
        #similar to what we just did above, but now checking if the type is a mira or lpv. if main or secondary
        #type indicate that the star is/could be an lpv or mira, we flag it
        if resm['OTYPE'][0] in micheck:
                mira_lpv_flag = np.append(mira_lpv_flag, 1)
        else:
                if any(param in resm['OTYPES'][0] for param in micheck_sec):
                        mira_lpv_flag = np.append(mira_lpv_flag, 1)
                else:
                        mira_lpv_flag = np.append(mira_lpv_flag, 0)
        #add both the name from the mdwarf dr3 table and the name from the xshooter spectra header to
        #our array of stellar names
        name_list = np.append(name_list, str(hdus[0].header['HNAME'])+' or '+ str(mdwarf_params[(mdwarf_params.RA.str.contains(hdus[0].header['RA_FULL']))|
                                (mdwarf_params.RA==hdus[0].header['RA_FULL'])].Star.values[0]))
        #find logg from the mdwarf dr3 table, add it to our array of logg info
        #(note I take the mean of logg, feh etc in case there are multiple obsevations/measurements)
        logg = mdwarf_params[(mdwarf_params.RA.str.contains(hdus[0].header['RA_FULL']))|
                                (mdwarf_params.RA==hdus[0].header['RA_FULL'])].logg.mean()
        loggs = np.append(loggs, logg)
        #some of the stars in the mdwarf dr3 table have '<NA>' values in their teff estimate. here, we
        #find teff from the mdwarf dr3 table and check if it is an actual value, or if it is '<NA>'.
        #if it is an actual value, we add it to the array of teffs, if it is not we just add 'nan' to the
        #array of teffs instead
        teff = mdwarf_params[(mdwarf_params.RA.str.contains(hdus[0].header['RA_FULL']))|
                                (mdwarf_params.RA==hdus[0].header['RA_FULL'])].Teff
        if teff.astype(str).str.contains('<NA>').any():
                teffs = np.append(teffs,np.NaN)
        else:
                teffs = np.append(teffs,teff.mean())
        #find logg from the mdwarf dr3 table, add it to our array of logg info
        feh = mdwarf_params[(mdwarf_params.RA.str.contains(hdus[0].header['RA_FULL']))|
                                (mdwarf_params.RA==hdus[0].header['RA_FULL'])]['[Fe/H]e'].mean()
        fehs = np.append(fehs, feh)
        #compute the synthetic g-i and nb - g color, add each color to the respective array of
        # synthetic colors
        g_i_color = np.append(g_i_color, synthcolor(hdus[1], filt_g, filt_i, dered=True))
        nb_g_color = np.append(nb_g_color, synthcolor(hdus[1], filt_nb515, filt_g, dered=True))
        #setting flags - all of the mdwarf stars should have values, so set values flag to zero
        #similarly, these stars don't have flags in their spectra
        values = np.append(values, 0)
        flag = np.append(flag, 0)
    #if there isn't 'M' in the xshooter id, the star is not a new mdwarf from the dr3 catalogue and we can
    #proceed using xshooter dr2 table
    else:
        #the HNAME in the xshooter spectra headers for the remaining stars do match the names given in
        #the dr2 table, so no need to do anything fancy here
        #add the name of the star to our array of names
        name_list = np.append(name_list, hdus[0].header['HNAME'])
        #query simbad for info on this star, using the HNAME
        res = customSimbad.query_object(hdus[0].header['HNAME'])
        #interestingly, not /all/ stars are in simbad. if nothing is found in simbad,
        #add 'NaN' to the array of main and secondary types, set the mira and variable flags to zero
        if type(res) == type(None):
                sp_type = np.append(sp_type, np.NaN)
                sec_sp_type = np.append(sec_sp_type, np.NaN)
                var_flag = np.append(var_flag, 0)
                mira_lpv_flag = np.append(mira_lpv_flag, 0)
        #if we get a result from simbad, add the 'OTYPE' to our array of types, and 'OTYPES' to secondary types array
        else:
                sp_type = np.append(sp_type, res['OTYPE'][0])
                sec_sp_type = np.append(sec_sp_type, res['OTYPES'][0])
                #again checking if the main type is variable, then checking if secondary type is variable. Set
                #flags accordingly
                if res['OTYPE'][0] in mainvar_set:
                        var_flag = np.append(var_flag, 1)
                        print(hdus[0].header['HNAME'], 'is variable')
                else:

                        if any(param in res['OTYPES'][0] for param in secvar_set):
                                var_flag = np.append(var_flag, 1)
                                print(hdus[0].header['HNAME'], 'is variable')
                        else:
                                var_flag = np.append(var_flag, 0)
                # checking if mira or LPV, same as above
                if res['OTYPE'][0] in micheck:
                        mira_lpv_flag = np.append(mira_lpv_flag, 1)
                else:
                        if any(param in res['OTYPES'][0] for param in micheck_sec):
                                mira_lpv_flag = np.append(mira_lpv_flag, 1)
                        else:
                                mira_lpv_flag = np.append(mira_lpv_flag, 0)
        #find the star in the table of stellar parameters from xshooter dr2
        #here, we use the fact that the HNAMES should match. first we check that this star is in the table
        #of stellar parameters at least once (some stars are not in this catalogue, see 'else' below)
        if len(stellar_params[(stellar_params['HNAME'].str.contains(hdus[0].header['HNAME']))
                |(stellar_params['HNAME']==hdus[0].header['HNAME'])]) > 0:
                #take the mean of the logg estimates for this star, add it to our array of logg
                logg = stellar_params[(stellar_params['HNAME'].str.contains(hdus[0].header['HNAME']))
                        |(stellar_params['HNAME']==hdus[0].header['HNAME'])].logg.mean()
                loggs = np.append(loggs, logg)
                #find the teff estimate for this star
                teff = stellar_params[(stellar_params['HNAME'].str.contains(hdus[0].header['HNAME']))
                |(stellar_params['HNAME']==hdus[0].header['HNAME'])].Teff
                #some stars have '<NA>' values for Teff. If the Teff is '<NA>', we set add nan to
                #the array of Teff. if it is a real value, we add the average of the Teff estimates
                #to the aray of Teff
                #note that I should improve this, should at least check if there is a non-<NA> estimate from a second observation
                if teff.astype(str).str.contains('<NA>').any():
                        print(teff)
                        teffs = np.append(teffs,np.NaN)
                else:
                        teffs = np.append(teffs,teff.mean())
                #take the mean of the feh estimates for this star, add it to our array of feh
                feh = stellar_params[(stellar_params['HNAME'].str.contains(hdus[0].header['HNAME']))
                        |(stellar_params['HNAME']==hdus[0].header['HNAME'])]['__Fe_H_'].mean()
                fehs = np.append(fehs, feh)
                #compute the synthetic g-i and nb - g color, add each color to the respective array of
                # synthetic colors
                g_i_color = np.append(g_i_color, synthcolor(hdus[1], filt_g, filt_i, dered=True))
                nb_g_color = np.append(nb_g_color, synthcolor(hdus[1], filt_nb515, filt_g, dered=True))
                #set values flag to zero
                values = np.append(values, 0)
                #check if there are comments for this star. if so, flag it
                if len(comment_flags[comment_flags['Star'].str.contains(hdus[0].header['HNAME'])])>0:
                        flag = np.append(flag, 1)
                else:
                        flag = np.append(flag, 0)
        #if there is no entry for this star in the dr2 table of stellar parameters, compute the color,
        #add NaN to the feh, teff, logg arrays. Set values flag to 1
        else:
                g_i_color = np.append(g_i_color, synthcolor(hdus[1], filt_g, filt_i, dered=True))
                nb_g_color = np.append(nb_g_color, synthcolor(hdus[1], filt_nb515, filt_g, dered=True))
                fehs = np.append(fehs, np.NaN)
                teffs = np.append(teffs, np.NaN)
                loggs = np.append(loggs, np.NaN)
                values = np.append(values, 1)
                if len(comment_flags[comment_flags['Star'].str.contains(hdus[0].header['HNAME'])])>0:
                        flag = np.append(flag, 1)
                else:
                        flag = np.append(flag, 0)
#%%
#note that some spectra have divide by zero errors - these stars will have one or both
#colors equal to inf
# %%
df = pd.DataFrame()
df['gi'] = g_i_color
df['nbg'] = nb_g_color
df['logg'] = loggs
df['feh'] = fehs
df['teff'] = teffs
df['name'] = name_list
df['xshoot_name'] = xshoot_name
df['main_type'] = sp_type
df['secondary_type'] = sec_sp_type
df['flag'] = flag
df['var_flag'] = var_flag
df['mira_lpv_flag'] = mira_lpv_flag
df['fits_flag'] = values
df.to_csv('xshooter_synthcolor.csv', index=False)
#%%
df[(df.nbg>5)] #looking at the stars with divide by zero errors-
#%%
print(len(df[(df.var_flag>0)&(df.logg<1)]), 'log(g) < 1 giants flagged as variable ',
        len(df[(df.var_flag<1)&(df.logg<1)]), 'log(g) < 1 non-variable')
#counting number of red giants that are miras, lpvs
print(len(df[(df.logg<4)&(df.gi>2)]), ' total giants with g - i > 2')
print(len(df[(df.logg<4)&(df.gi>2)&(df.var_flag>0)]), ' of these very red giants are variable')
print(len(df[(df.logg<4)&(df.gi>2)&(df.mira_lpv_flag>0)]), ' of these very red giants are mira or LPV')
#%%
len(df[(df.var_flag>0)]), len(df[df.mira_lpv_flag>0])
#%%
#from https://ui.adsabs.harvard.edu/abs/2002AJ....124.1706V/abstract
#we have an observational estimate of the radii of a few Miras at different points in their periods
#which we can use to estimate how much log(g) changes, as log(g) \propto log(1/r^2)
#here we have theoretical predictions for R changes https://ui.adsabs.harvard.edu/abs/2011MNRAS.418..114I/abstract
#this paper also gives log(g) vs phase https://articles.adsabs.harvard.edu//full/1998IAPPP..73...59P/0000072.000.html
#but isn't cited at all / seems to exist as an island, hard to gauge accuracy since they
#fit models to estimate log(g)
# ex one star has changes of ~260 to ~330 Rsun, another star from ~440 to ~410 rsun,
#model predictions from ~0.85 'parent radii' to ~1.4 'parent radii'
#and so on-
#over all, find that miras/LPVs should have log(g) change of like less than 1 but can be like
#delta log(g) of ~.3 or ~.4
#%%
# %%
#reading in HSC UMi data to plot as background
combined = pd.read_csv('/home/cfilion1/Galaxia/ursaminor_hsc_cleaned.csv')
fig, (ax1, ax2) = plt.subplots(1,2, figsize=(24,10))
ax1.scatter(combined.gpsf-combined.ipsf,
        combined.npsf-combined.gpsf, c='grey',s=.1,rasterized=True)
#plotting synthetic colors for the x-shooter stars colored by logg, first plot non-variables
c1 = ax1.scatter(df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag<1)]['gi'],
                df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag<1)]['nbg'],
                c=df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag<1)]['logg'], vmin=0, vmax=5., cmap='rainbow')
#then plot variables that are not lpv/mira
g = ax1.scatter(df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag>0)&(df.mira_lpv_flag<1)]['gi'],
                df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag>0)&(df.mira_lpv_flag<1)]['nbg'],
                c=df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag>0)&(df.mira_lpv_flag<1)]['logg'],
                vmin=0, vmax=5., cmap='rainbow',
                marker='s', label = 'Variable')
#finally, plot lpvs/miras
q = ax1.scatter(df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag>0)&(df.mira_lpv_flag>0)]['gi'],
                df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag>0)&(df.mira_lpv_flag>0)]['nbg'],
                c=df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag>0)&(df.mira_lpv_flag>0)]['logg'],
                vmin=0, vmax=5., cmap='rainbow',
                marker='*', s=100, label='LPV or Mira')
g.set_facecolor('none')
g.set_edgecolor('k')
q.set_facecolor('none')
q.set_edgecolor('k')
ax1.legend()
ax1.set_xlim(-0.75, 4)
ax1.set_ylim(0.8,-.8)
ax1.set_xlabel('$(g-i)_0$')
ax1.set_ylabel('$(NB515 - g)_0$')

ax2.scatter(combined.gpsf-combined.ipsf,
        combined.npsf-combined.gpsf, c='grey',s=.1,rasterized=True)
#same as above, but color-coding by feh
c2 = ax2.scatter(df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag<1)]['gi'],
                df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag<1)]['nbg'],
                c=df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag<1)]['feh'], vmin=-3.5, vmax=1, cmap='jet')

gg = ax2.scatter(df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag>0)&(df.mira_lpv_flag<1)]['gi'],
                df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag>0)&(df.mira_lpv_flag<1)]['nbg'],
                c=df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag>0)&(df.mira_lpv_flag<1)]['feh'],
                vmin=-3.5, vmax=1, cmap='jet', marker='s', label='Variable')
qq = ax2.scatter(df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag>0)&(df.mira_lpv_flag>0)]['gi'],
                df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag>0)&(df.mira_lpv_flag>0)]['nbg'],
                c=df[(df.flag<1)&(df.fits_flag<1)&(df.var_flag>0)&(df.mira_lpv_flag>0)]['feh'],
                vmin=-3.5, vmax=1, cmap='jet', marker='*', s=100, label='Mira or LPV')
gg.set_facecolor('none')
gg.set_edgecolor('k')
qq.set_facecolor('none')
qq.set_edgecolor('k')
ax2.legend()
ax2.set_xlim(-0.75, 4)
ax2.set_ylim(0.8,-.8)
ax2.set_xlabel('$(g-i)_0$')
ax2.set_ylabel('$(NB515 - g)_0$')

ax1.set_title('Colored by log(g)')
ax2.set_title('Colored by [Fe/H]')
fig.colorbar(c1, ax=ax1, orientation = 'vertical', label='log(g)')
fig.colorbar(c2, ax=ax2, orientation = 'vertical', label='[Fe/H]')
plt.subplots_adjust(wspace=.2)
plt.savefig('/home/cfilion1/xshooter_cc_diagram.png',bbox_inches = 'tight',
    pad_inches = 0.1)
#%%

#%%
#plotting teff vs color, teff vs logg
fig, (ax1, ax2) = plt.subplots(1,2, figsize=(24,10))
c1 = ax1.scatter(np.log10(df[(df.flag<1)&(df.fits_flag<1)&(df.nbg<99)&(df.var_flag<1)]['teff'].astype(float)),
        df[(df.flag<1)&(df.fits_flag<1)&(df.nbg<99)&(df.var_flag<1)]['gi'],
        c=df[(df.flag<1)&(df.fits_flag<1)&(df.nbg<99)&(df.var_flag<1)]['logg'],
        cmap='copper')
cc = ax1.scatter(np.log10(df[(df.flag<1)&(df.fits_flag<1)&(df.nbg<99)&(df.var_flag>0)]['teff'].astype(float)),
        df[(df.flag<1)&(df.fits_flag<1)&(df.nbg<99)&(df.var_flag>0)]['gi'],
        c=df[(df.flag<1)&(df.fits_flag<1)&(df.nbg<99)&(df.var_flag>0)]['logg'],
        cmap='copper', marker='s')
cc.set_facecolor('none')
fig.colorbar(c1, ax=ax1, orientation = 'vertical', label='log(g)')
ax1.set_xlabel('log(Teff)')
ax1.set_xlim(np.log10(21000), np.log10(2100))
ax1.set_ylabel('g-i')
ax1.set_ylim(-1.0,4.5)

c2 = ax2.scatter(np.log10(df[(df.flag<1)&(df.fits_flag<1)&(df.nbg<99)&(df.var_flag<1)]['teff'].astype(float)),
        df[(df.flag<1)&(df.fits_flag<1)&(df.nbg<99)&(df.var_flag<1)]['logg'],
        c=df[(df.flag<1)&(df.fits_flag<1)&(df.nbg<99)&(df.var_flag<1)]['nbg'],
        cmap='rainbow')
aa = ax2.scatter(np.log10(df[(df.flag<1)&(df.fits_flag<1)&(df.nbg<99)&(df.var_flag>0)]['teff'].astype(float)),
        df[(df.flag<1)&(df.fits_flag<1)&(df.nbg<99)&(df.var_flag>0)]['logg'],
        c=df[(df.flag<1)&(df.fits_flag<1)&(df.nbg<99)&(df.var_flag>0)]['nbg'],
        cmap='rainbow', marker='s')
aa.set_facecolor('none')
fig.colorbar(c2, ax=ax2, orientation = 'vertical', label='NB515-g')
ax2.set_xlabel('log(Teff)')
ax2.set_xlim(np.log10(21000), np.log10(2100))
ax2.set_ylabel('log(g)')
ax2.set_ylim(5.5,-.2)
plt.subplots_adjust(wspace=.1)
plt.savefig('/home/cfilion1/giteff_loggteff.png')
#%%
#%%
#looking at stars with feh > 0
fig, (ax1, ax2) = plt.subplots(1,2, figsize=(24,10))
ax1.scatter(combined.gpsf-combined.ipsf,
        combined.npsf-combined.gpsf, c='grey',s=.1,rasterized=True)
c1 = ax1.scatter(df[(df.feh>0)]['gi'], df[(df.feh>0)]['nbg'],
                c=df[(df.feh>0)]['logg'], vmin=0, vmax=5., cmap='rainbow')
ax1.set_xlim(-0.75, 4)
ax1.set_ylim(0.8,-.8)
ax1.set_xlabel('$(g-i)_0$')
ax1.set_ylabel('$(NB515 - g)_0$')

ax2.scatter(combined.gpsf-combined.ipsf,
        combined.npsf-combined.gpsf, c='grey',s=.1,rasterized=True)

c2 = ax2.scatter(df[(df.feh>0)]['gi'], df[(df.feh>0)]['nbg'],
                c=df[(df.feh>0)]['feh'], vmin=0, vmax=1, cmap='rainbow')

ax2.set_xlim(-0.75, 4)
ax2.set_ylim(0.8,-.8)
ax2.set_xlabel('$(g-i)_0$')
ax2.set_ylabel('$(NB515 - g)_0$')

fig.colorbar(c1, ax=ax1, orientation = 'vertical', label='log(g)')
fig.colorbar(c2, ax=ax2, orientation = 'vertical', label='[Fe/H]')
plt.subplots_adjust(wspace=.2)
plt.savefig('/home/cfilion1/metalrich_xshooter_cc_diagram.png')

#looking at stars with feh < -2
fig, (ax1, ax2) = plt.subplots(1,2, figsize=(24,10))
fig.suptitle('Stars with [Fe/H] < -2')
ax1.scatter(combined.gpsf-combined.ipsf,
        combined.npsf-combined.gpsf, c='grey',s=.1,rasterized=True)
c1 = ax1.scatter(df[(df.feh<-2)]['gi'], df[(df.feh<-2)]['nbg'],
                c=df[(df.feh<-2)]['logg'], vmin=0, vmax=5., cmap='rainbow')
ax1.set_xlim(-0.75, 4)
ax1.set_ylim(0.8,-.8)
ax1.set_xlabel('$(g-i)_0$')
ax1.set_ylabel('$(NB515 - g)_0$')

ax2.scatter(combined.gpsf-combined.ipsf,
        combined.npsf-combined.gpsf, c='grey',s=.1,rasterized=True)

c2 = ax2.scatter(df[(df.feh<-2)]['gi'], df[(df.feh<-2)]['nbg'],
                c=df[(df.feh<-2)]['feh'], vmin=-3.5, vmax=-2, cmap='rainbow')

ax2.set_xlim(-0.75, 4)
ax2.set_ylim(0.8,-.8)
ax2.set_xlabel('$(g-i)_0$')
ax2.set_ylabel('$(NB515 - g)_0$')

fig.colorbar(c1, ax=ax1, orientation = 'vertical', label='log(g)')
fig.colorbar(c2, ax=ax2, orientation = 'vertical', label='[Fe/H]')
plt.subplots_adjust(wspace=.2)
plt.savefig('/home/cfilion1/metalpoor_xshooter_cc_diagram.png')
#%%
#%%
# %%
#OGLEII DIA BUL-SC22 1319 is mira variable, no nb mag
#SHV 0534578-702532 is mira variable
#SHV 0517337-725738 is carbon star, candidate lpv
#SHV 0518161-683543 is carbon star, also lpv/maybe mira
#OGLEII DIA BUL-SC03 3941 is mira variable

# %%

#%%