#!/usr/bin/env python
__author__ = "solivr"
__license__ = "GPL"

import json
import numpy as np
from collections import OrderedDict
from tqdm import tqdm
from glob import glob
from imageio import imread, imsave
import os

MAP_COLORS = OrderedDict([('background', ((0, 0, 1), (0, 0, 0))),
                          ('comment', ((0, 0, 2), (0, 0, 255))),
                          ('decoration', ((0, 0, 4), (0, 255, 0))),
                          ('text', ((0, 0, 8), (255, 0, 0))),
                          ('comment_deco', ((0, 0, 6), (0, 255, 255))),
                          ('text_comment', ((0, 0, 10), (255, 0, 255))),
                          ('text_deco', ((0, 0, 12), (255, 255, 0)))])

def parse_diva_tool_output(score_txt: str, output_json_filename: str=None)-> dict:
    """
    This fn parses the output of JAR DIVA Evaluation tool
    :param score_txt: filename of txt score containing output of DIVA evaluation tool
    :param output_json_filename: filename to output the parsed result in json
    :return: dict containing the parsed results
    """
    def process_hlp_fn(string):
        """
        Processes format : R=0.64,0.52
        """
        key, vals = string.split('=')
        vals = vals.split(',')
        return {key: float(vals[0]), key + '_fw': float(vals[1])}, key

    def process_hlp_per_class_format(string, measure_key):
        """
        Processes format : '0.26|0.77|0.77|0.77
        '"""
        return {measure_key: [float(t) for t in string.split('|')]}

    lines = score_txt.splitlines()

    dic_results = {'Mean_IU': float(lines[0].split(' = ')[1])}
    measures = lines[1].split(' ')
    dic_results = {**dic_results, **{m.split('=')[0]: float(m.split('=')[1]) for m in measures[:2]}}
    for m in measures[2:5]:
        eq, tab = m.split('[')
        dic, key = process_hlp_fn(eq)
        dic_results = {**dic_results, **dic}
        dic_results = {**dic_results, **process_hlp_per_class_format(tab[:-1], key + '_per_class')}
    sp = measures[-1].split('[')
    dic, key = process_hlp_fn(sp[0])
    dic_results = {**dic_results, **dic}
    dic_results = {**dic_results, **process_hlp_per_class_format(sp[1][:-6], key + '_per_class')}
    dic_results = {**dic_results, **process_hlp_per_class_format(sp[2][:-1], sp[1][-5:-1])}

    if output_json_filename is not None:
        with open(output_json_filename, 'w') as f:
            json.dump(dic_results, f)

    return dic_results

def to_original_color_code(bin_prediction):
    """
    (0,0,0) : Background
    (255,0,0) : Text
    (0,255,0) : decoration
    (0,0,255) : comment

    RGB=0x000008: main text body
    RGB=0x000004: decoration
    RGB=0x000002: comment
    RGB=0x000001: background
    RGB=0x00000A: main text body+comment
    RGB=0x00000C: main text body+decoration
    RGB=0x000006: comment +decoration

    :param bin_prediction:
    :return:
    """
    pred_original_colors = np.zeros_like(bin_prediction)
    for key, colors in MAP_COLORS.items():
        pred_original_colors[np.all(bin_prediction == colors[1], axis=-1)] = colors[0]

    return pred_original_colors

def diva_dataset_generator(input_dir: str, output_dir: str):
    """

    :param input_dir: Input directory containing images and PAGE files
    :param output_dir: Output directory to save images and labels
    :return:
    """

    img_filenames = glob(os.path.join(input_dir, 'img', '*.jpg'))
    output_img_dir = os.path.join(output_dir, 'images')
    output_label_dir = os.path.join(output_dir, 'labels')

    def annotate_one(gt_image: np.array, map_colors: dict=MAP_COLORS):
        label_img = np.zeros_like(gt_image)
        for key, colors in map_colors.items():
            label_img[np.all(gt_image == colors[0], axis=-1)] = colors[1]

        return label_img

    for filename in tqdm(img_filenames):
        img = imread(filename, pilmode='RGB')
        basename = os.path.basename(filename).split('.')[0]
        filename_label = os.path.join(input_dir, 'pixel-level-gt', '{}.png'.format(basename))
        gt_img = imread(filename_label, pilmode='RGB')
        label_image = annotate_one(gt_img, MAP_COLORS)

        # Save
        imsave(os.path.join(output_img_dir, '{}.jpg'.format(basename)), img)
        imsave(os.path.join(output_label_dir, '{}.png'.format(basename)), label_image)


import logging

from objective_turk import objective_turk

logger = logging.getLogger(__name__)

EXTERNAL_URL_QUESTION = """<?xml version="1.0"?>
<ExternalQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd">
    <ExternalURL>{}</ExternalURL>
    <FrameHeight>600</FrameHeight>
</ExternalQuestion>
"""

HTML_QUESTION = """<?xml version="1.0"?>
<HTMLQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2011-11-11/HTMLQuestion.xsd">
<HTMLContent><![CDATA[
{}
]]></HTMLContent>
<FrameHeight>0</FrameHeight>
</HTMLQuestion>
"""

def get_external_question(url: str):
    """
    Return a Question string for an External URL HIT pointing to the given URL.
    """
    return EXTERNAL_URL_QUESTION.format(url)

def get_html_question(html: str):
    """
    Return a Question string for an HTMLQuestion with the given content.
    """
    return HTML_QUESTION.format(html)

class BuiltinQualificationType:
    """
    An Enum for QualificationTypeId constants
    https://github.com/nmalkin/mturk-python/blob/master/mturk/mturk.py#L16
    """

    P_SUBMITTED = "00000000000000000000"
    P_ABANDONED = "00000000000000000070"
    P_RETURNED = "000000000000000000E0"
    P_APPROVED = "000000000000000000L0"
    P_REJECTED = "000000000000000000S0"
    N_APPROVED = "00000000000000000040"
    LOCALE = "00000000000000000071"
    ADULT = "00000000000000000060"
    S_MASTERS = "2ARFPLSP75KLA8M8DH1HTEQVJT3SY6"
    MASTERS = "2F1QJWKUDD8XADTFD2Q0G6UTO95ALH"
    S_CATMASTERS = "2F1KVCNHMVHV8E9PBUB2A4J79LU20F"
    CATMASTERS = "2NDP2L92HECWY8NS8H3CK0CP5L9GHO"
    S_PHOTOMASTERS = "2TGBB6BFMFFOM08IBMAFGGESC1UWJX"
    PHOTOMASTERS = "21VZU98JHSTLZ5BPP4A9NOBJEK3DPG"

MINIMUM_PERCENTAGE_APPROVED = 95

def get_qualifications(exclude: str = None, include: str = None):
    qualifications = [
        {
            'QualificationTypeId': BuiltinQualificationType.LOCALE,
            'Comparator': 'EqualTo',
            'LocaleValues': [{'Country': 'US'}],
            'RequiredToPreview': True,
        },
        {
            'QualificationTypeId': BuiltinQualificationType.P_APPROVED,
            'Comparator': 'GreaterThan',
            'IntegerValues': [MINIMUM_PERCENTAGE_APPROVED],
            'RequiredToPreview': True,
        },
    ]

    if exclude is not None:
        for qualification_id in exclude:
            logging.debug('excluding workers with qualification %s', qualification_id)
            qualifications.append(
                {
                    'QualificationTypeId': qualification_id,
                    'Comparator': 'DoesNotExist',
                    'RequiredToPreview': True,
                }
            )

    if include is not None:
        for qualification_id in include:
            logging.debug(
                'allowing only workers with qualification %s', qualification_id
            )
            qualifications.append(
                {
                    'QualificationTypeId': qualification_id,
                    'Comparator': 'Exists',
                    'RequiredToPreview': True,
                }
            )

    return qualifications

def create_hit_with_hit_type(**kwargs):
    """
    Create HIT using provided HITTypeId.

    You still need to pass 'LifetimeInSeconds', 'MaxAssignments', 'Question'.

    Full list of valid parameters:
    HITTypeId, MaxAssignments, LifetimeInSeconds, Question, RequesterAnnotation, UniqueRequestToken, AssignmentReviewPolicy, HITReviewPolicy, HITLayoutId, HITLayoutParameters

    Other fields will be ignored:
    Title, Description, Reward, and Keywords
    """
    if 'HITTypeId' not in kwargs:
        raise ValueError('missing required argument HITTypeId')
    elif 'Question' not in kwargs:
        raise ValueError('missing required argument Question')
    elif 'MaxAssignments' not in kwargs:
        raise ValueError('missing required argument MaxAssignments')

    hit_type = kwargs['HITTypeId']
    logger.info(
        'creating HIT using HITTypeId %s. Title, Description, Reward, and Keywords from calling script will be ignored.',
        hit_type,
    )
    response = objective_turk.client().create_hit_with_hit_type(**kwargs)
    logger.debug(response)
    #pylint: disable=protected-access
    return objective_turk.Hit._new_from_response(response['HIT'])

def create_hit(**kwargs):
    """
    Create a HIT with the given arguments.

    For arguments, see:
    https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/mturk.html#MTurk.Client.create_hit
    """
    response = objective_turk.client().create_hit(**kwargs)
    logger.debug(response)
    #pylint: disable=protected-access
    return objective_turk.Hit._new_from_response(response['HIT'])


# H-Index

def solution(citations):
    answer = 0
    citations.sort(reverse=True)
    count = 0

    for i in range(0, len(citations) + 1):
        for j in citations:
            if i <= j:
                count += 1
            if i == count:
                answer = i
        count = 0

    return answer

citations1 = [3, 0, 6, 1, 5]
citations2 = [5, 5, 5, 5]
print(solution(citations1))
print(solution(citations2))


from dimod import ConstrainedQuadraticModel, Binaries, ExactCQMSolver, quicksum, Binary
from dwave.system import LeapHybridCQMSampler
import numpy as np

#example ffrom dwave documentation: https://docs.ocean.dwavesys.com/en/stable/docs_dimod/reference/models.html?highlight=cqm#module-dimod.constrained.constrained
#the obective if minimize the number of used bins while not exceding the capacity of each bin
#weights of packs
weights = [.9, .7, .2, .1]
#capacity of each bin
capacity = 2
#variables are for bins: if yj=1 means that bin j is used (note that we suppose number of bins = number of packs)
y = [Binary(f'y_{j}') for j in range(len(weights))]
#xij are variables that indicate if a pack i is put in bin j
x = [[Binary(f'x_{i}_{j}') for j in range(len(weights))]
     for i in range(len(weights))]

#create empty cqm
cqm=ConstrainedQuadraticModel()

#objective
cqm.set_objective(sum(y))

#constraints

#constraint1: each pack must be put at exactly one bin
for i in range(len(weights)):
    cqm.add_constraint(sum(x[i]) == 1, label=f'item_placing_{i}')

#constraint2: weights of packs put in a bin cannot exceed bin's capacity
for j in range(len(weights)):
    cqm.add_constraint(
        sum(weights[i] * x[i][j] for i in range(len(weights))) - y[j] * capacity <= 0,
        label=f'capacity_bin_{j}')

#variant with additional constraint from me
#constraint3: packs with big different weights cannot be in the same bin (crieteria w[i]-w[k]<0.3)
#cqm.add_constraint(quicksum(x[i][j]*x[k][j] for j in range(len(weights)) for i in range(len(weights)-1) for k in range(i+1, len(weights)) if (abs(weights[i]-weights[k])>=0.3))==0, label=f'packs with small wieghts diff in the same bin')

#solve
sampleset = ExactCQMSolver().sample_cqm(cqm)
all_samples=sampleset.samples
#print(all_samples)
feasible_sampleset = sampleset.filter(lambda row: row.is_feasible)
fsamples_agr=feasible_sampleset.aggregate()
print('---------------------------------------------FEASIBLE-------------------------------------------------------------')
s=feasible_sampleset.first
print(s)


from flask import Blueprint, render_template, request, redirect, url_for, session, g, flash, current_app
# from werkzeug.utils import redirect
from sqlalchemy import func, nullslast
from datetime import datetime
from .auth_views import login_required
from ..forms import QuestionForm, AnswerForm
from ..models import Question, User
from .. import db

bp = Blueprint("question", __name__, url_prefix="/question")

@bp.before_request
def load_logged_in_user():
    user_id = session.get("user_id")
    if user_id is None:
        g.user = None
    else:
        g.user = User.query.get(user_id)

def _nullslast(obj):
    if current_app.config["SQLALCHEMY_DATABASE_URI"].startswith("sqlite"):
        return obj
    else:
        return nullslast(obj)

@bp.route("/list/")
def list_():
    page_num = request.args.get("page", type=int, default=1)
    # question_list = Question.query.order_by(Question.created_at.desc())                 # BaseQuery obj
    # question_list = question_list.paginate(page_num, per_page=10).items                 # list obj
    # return render_template("question/question_list.html", question_list=question_list)  # list obj
    # question_query = Question.query.order_by(Question.created_at.desc())                # BaseQuery obj
    question_query = Question.query.order_by(Question.created_at.desc())                  # BaseQuery obj
    question_page = question_query.paginate(page_num, per_page=10)                        # Pagination obj
    return render_template("question/question_list.html", question_page=question_page)    # list obj

@bp.route("/detail/<int:question_id>/")
def detail(question_id):  # detail_question_with_answer
    form = AnswerForm()
    # question = Question.query.get(question_id)
    question = Question.query.get_or_404(question_id)
    # question_detail needs question object
    return render_template("question/question_detail.html", question=question, form=form)

@bp.route("/create/", methods=["GET", "POST"])
@login_required
def create():  # create_question
    form = QuestionForm()
    if request.method == "POST" and form.validate_on_submit():
        question = Question(subject=form.subject.data, content=form.content.data, created_at=datetime.now(), user=g.user)
        db.session.add(question)
        db.session.commit()
        return redirect(url_for("main.index"))
    return render_template("question/question_form.html", form=form)

@bp.route("/modify/<int:question_id>", methods=["GET", "POST"])
@login_required
def modify(question_id):
    question = Question.query.get_or_404(question_id)
    if g.user != question.user:
        flash("수정 권한이 없습니다.")
        return redirect(url_for("question.detail", question_id=question_id))

    if request.method == "POST":
        form = QuestionForm()
        if form.validate_on_submit():
            # fill up the question object
            form.populate_obj(question)
            question.modified_at = datetime.now()
            db.session.commit()
            return redirect(url_for("question.detail", question_id=question_id))
    else:  # "GET"
        form = QuestionForm(obj=question)
    return render_template("question/question_form.html", form=form)

@bp.route("/delete/<int:question_id>")
@login_required
def delete(question_id):
    question = Question.query.get_or_404(question_id)
    if g.user != question.user:
        flash("삭제 권한이 없습니다.")
        return redirect(url_for("question.detail", question_id=question_id))

    db.session.delete(question)
    db.session.commit()
    return redirect(url_for("question.list_"))


class Solution:
    """
    @param color: the given color
    @return: a 7 character color that is most similar to the given color
    """
    def similarRGB(self, color):
        return "#" + self.similarChannel(color[1:3]) + \
            self.similarChannel(color[3:5]) + self.similarChannel(color[5:])

    def similarChannel(self, c):
        # channel is a 2-char string
        c_dec = int(c, 16)
        candidate = int(c[0] + c[0], 16) # a possibility
        difference = candidate - c_dec
        if difference <= 0x9 and difference > -0x9:
            return hex(candidate)[2:] if candidate != 0x0 else "00"
        elif difference > 0x9:
            return hex(candidate - 0x11)[2:] if (candidate != 0x0 and candidate - 0x11 != 0x0) else "00"
        else:
            return hex(candidate + 0x11)[2:]

s = Solution()

a = "#0f166"
print(s.similarRGB(a))


Z_POWER = 2
DEPTH = 10000
SCALE = 1
XOFFSET, YOFFSET = 0, 0

# # The function to approximate
# def realFunc(x, y):
#     z = 0
#     c = x + y*1j
#
#     for i in range(DEPTH, 0, -1):
#         if z.real > 2:
#             return i
#         else:
#             z = (z ** Z_POWER) + (c/SCALE) + XOFFSET + (YOFFSET*1j)
#     return 0

# The function to approximate
# Sum two complex numbers
def realFunc(x1, x2):
    z = x1 + x2*1j
    z = z ** 2
    return z.real, z.imag


from __future__ import unicode_literals

from django.db import models
from django.utils.translation import ugettext as _
from django.utils.encoding import python_2_unicode_compatible

@python_2_unicode_compatible
class Location(models.Model):
    """
    Location.
    """
    name = models.CharField(
        _('name'),
        help_text=_('Name of the location.'),
        max_length=255,
        blank=False,
        null=False
    )
    slug = models.SlugField(
        _('slug'),
        help_text=_('Location name as used in links.'),
        max_length=255,
        blank=False,
        null=False,
        unique=True
    )
    address = models.CharField(
        _('address'),
        help_text=_('Location address.'),
        max_length=255,
        blank=False,
        null=False
    )

    class Meta:
        verbose_name = _('location')
        verbose_name_plural = _('locations')

    def __str__(self):
        return self.name


"""empty message

Revision ID: 9c40a446cd09
Revises: 0b2029b52fcc
Create Date: 2020-05-13 08:23:56.700811

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '9c40a446cd09'
down_revision = '0b2029b52fcc'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.alter_column('artist', 'seeking_talent',
               existing_type=sa.BOOLEAN(),
               nullable=False)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.alter_column('artist', 'seeking_talent',
               existing_type=sa.BOOLEAN(),
               nullable=True)
    # ### end Alembic commands ###


#
# @lc app=leetcode id=200 lang=python3
#
# [200] Number of Islands
#
# https://leetcode.com/problems/number-of-islands/description/
#
# algorithms
# Medium (45.70%)
# Likes:    4942
# Dislikes: 185
# Total Accepted:    668.3K
# Total Submissions: 1.5M
# Testcase Example:  '[["1","1","1","1","0"],["1","1","0","1","0"],["1","1","0","0","0"],["0","0","0","0","0"]]'
#
# Given a 2d grid map of '1's (land) and '0's (water), count the number of
# islands. An island is surrounded by water and is formed by connecting
# adjacent lands horizontally or vertically. You may assume all four edges of
# the grid are all surrounded by water.
#
# Example 1:
#
#
# Input:
# 11110
# 11010
# 11000
# 00000
#
# Output: 1
#
#
# Example 2:
#
#
# Input:
# 11000
# 11000
# 00100
# 00011
#
# Output: 3
#
#

# @lc code=start
class Solution:
    def numIslands(self, grid: List[List[str]]) -> int:
        # def bfs(i, j, row, col):
        #     grid[i][j] = '2'
        #     if i < row - 1 and grid[i + 1][j] == '1':
        #         bfs(i + 1, j, row, col)
        #     if i and grid[i - 1][j] == '1':
        #         bfs(i - 1, j, row, col)
        #     if j < col - 1 and grid[i][j + 1] == '1':
        #         bfs(i, j + 1, row, col)
        #     if j and grid[i][j - 1] == '1':
        #         bfs(i, j - 1, row, col)
        def bfs(i, j, vectors):
            if i < 0 or j < 0 or i >= len(grid) or j >= len(grid[0]):
                return
            if grid[i][j] != '1': return
            grid[i][j] = '2'
            for m, n in vectors:
                x = i + m
                y = j + n
                bfs(x, y, vectors)

        ans = 0
        row = len(grid)
        if not row: return ans
        col = len(grid[0])
        vectors = [(1, 0), (-1, 0), (0, 1), (0, -1)]
        for i in range(row):
            for j in range(col):
                if grid[i][j] == '1':
                    ans += 1
                    bfs(i, j, vectors)

        return ans

# @lc code=end


from io import StringIO

from pyNastran.bdf.field_writer_8 import print_card_8
from pyNastran.bdf.field_writer_16 import print_card_16
from pyNastran.bdf.cards.base_card import expand_thru
from pyNastran.bdf.bdf_interface.assign_type import integer

def get_spcadd_constraint(card):
    constraint_id = integer(card, 1, 'constraint_id')

    node_ids = card.fields(2)
    node_ids = expand_thru(node_ids)

    assert isinstance(constraint_id, int), constraint_id
    return constraint_id, node_ids

class SPCADD:
    """
    Defines a single-point constraint set as a union of single-point constraint
    sets defined on SPC or SPC1 entries.

    +--------+----+----+-----+
    |    1   | 2  |  3 |  4  |
    +========+====+====+=====+
    | SPCADD  | 2 | 1 |   3  |
    +---------+---+---+------+
    """
    type = 'SPCADD'
    def __init__(self, model):
        self.model = model
        self.spc_id = None
        self.spc_ids = []

    def add(self, spc_id, spc_ids, comment):
        #if comment:
            # self.comment = comment
        assert isinstance(spc_id, int), spc_id
        self.spc_id = spc_id
        self.spc_ids += spc_ids

    def build(self):
        self.spc_ids.sort()

    def write_card(self, bdf_file, size=8):
        card = ['SPCADD', self.spc_id] + self.spc_ids
        #print("card = ", card)
        if size == 8:
            bdf_file.write(print_card_8(card))
        else:
            bdf_file.write(print_card_16(card))

    def __repr__(self):
        f = StringIO()
        self.write_card(f)
        return f.getvalue().rstrip()


#!/usr/bin/env python3

##########################################################################################
# FILE CONTENTS:
# makes the 2-D squeezing figures for the SOC/squeezing paper
##########################################################################################

import os, sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from scipy import interpolate
from matplotlib import gridspec

from correlator_methods import squeezing_ops
from squeezing_methods import squeezing_from_correlators
from sr87_olc_constants import recoil_energy_NU

confining_depth = 60
data_dir = "../data/"
fig_dir = "../figures/"

OAT, TAT, TNT = "OAT", "TAT", "TNT"
NOAT = TAT # "not one-axis twisting"
sqz_methods = [ OAT, NOAT ]

depth_min, depth_max = 2, 7

depths_dec = np.arange(20,81,2)/10 # lattice depths in plots
sizes_dec = np.arange(10,101,5) # linear lattice sizes in plots
phi_cutoff = 10 # minimum value of (\phi/\pi)^{-1}

dpi = 600

params = { "font.family" : "sans-serif",
           "font.serif" : "Computer Modern",
           "text.usetex" : True }
plt.rcParams.update(params)

# define some common axis labels that we will use several times
depth_label = r"Lattice depth ($V_0/E_{\mathrm{R}}$)"
size_label = r"Linear lattice size ($\ell$)"
sqz_label = r"Squeezing (dB)"
time_label = r"Time"
U_J_label = r"$U/J$"
phi_label = r"$\log_{10}(\phi/\pi)$"

# read in 1-D or 2-D data files
def pd_read_1D(fname):
    return pd.read_csv(fname, comment = "#", squeeze = True, header = None, index_col = 0)
def pd_read_2D(fname):
    data = pd.read_csv(fname, comment = "#", header = 0, index_col = 0)
    data.columns = pd.to_numeric(data.columns)
    return data

J_0 = pd_read_1D(data_dir + "J_0.txt")
U_int = pd_read_2D(data_dir + "U_int_2D.txt")
all_depths = J_0.index
U_J = (U_int.T / J_0.T).T[confining_depth]
U_J_interp = interpolate.interp1d(U_J.values, all_depths)
U_J_ticks = np.array(range(int(U_J.values.max()))) + 1
depth_U_ticks = [ float(U_J_interp(val)) for val in U_J_ticks ]

# make a secondary axis marking values of U/J (against lattice depths)
def make_U_axis(axis):
    ax = axis.twiny()
    ax.set_xlim(all_depths[0],all_depths[-1])
    ax.set_xticks(depth_U_ticks)
    ax.set_xticklabels(U_J_ticks)
    ax.set_xlabel(U_J_label)
    ax.set_xlim(depth_min, depth_max)
    return ax

# add text to a sub-plot
method_box = dict(boxstyle = "round", facecolor = "white", alpha = 1)
def add_text(axis, text):
    axis.text(0.9, 0.1, text, transform = axis.transAxes, bbox = method_box,
              verticalalignment = "bottom", horizontalalignment = "right")

def to_dB(vals): return 10*np.log10(vals) # convert values to decibels

##########################################################################################
# squeezing without decoherence
##########################################################################################

figsize = (6.5,3)

# set up figure panel and linestyles
ax = {}
fig, ( ax_sqz, ax[OAT], ax[NOAT], cax ) \
    = plt.subplots(figsize = figsize, ncols = 4,
                   gridspec_kw = { "width_ratios" : [1,2,2,0.1] })
lines = { OAT : "k-",
          NOAT : "k--" }

# exctract data to plot
sqz_vals = {}
t_opt = {}
for method in sqz_methods:
    sqz_opt = -to_dB(pd_read_1D(data_dir + f"sqz_{method}.txt"))
    t_opt[method] = pd_read_2D(data_dir + f"optimization/t_opt_{method}_L_2D.txt")
    t_opt[method] /= recoil_energy_NU
    depths, sizes = t_opt[method].index, np.array(t_opt[method].columns)

    err = abs(depths[1] - depths[0]) / 2
    use_depths = (depths >= depth_min-err) & (depths <= depth_max+err)

    sqz_interp = interpolate.interp1d(sqz_opt.index, sqz_opt.values)
    sqz_vals[method] = np.array([ float(sqz_interp(L**2)) for L in sizes ])

sqz_min = min([ sqz_vals[method].min() for method in sqz_methods ])
sqz_max = max([ sqz_vals[method].max() for method in sqz_methods ])
t_min = min([ t_opt[method].values.min() for method in sqz_methods ])
t_max = max([ t_opt[method].values.max() for method in sqz_methods ])

faster_NOAT = t_opt[NOAT] > t_opt[OAT]

# plot data
for method in sqz_methods:
    ax_sqz.plot(sqz_vals[method], sizes, lines[method], label = method)

    data = t_opt[method].values[use_depths,:].T
    mesh = ax[method].pcolormesh(depths[use_depths], sizes, data,
                                 cmap = plt.get_cmap("twilight_shifted"),
                                 vmin = 0, vmax = t_max,
                                 zorder = 0, rasterized = True)

    ax[method].contour(depths[use_depths], sizes,
                       faster_NOAT.values[use_depths,:].T,
                       levels = 0, colors = [ "k" ], linestyles = "dotted")

    ax[method].set_xlabel(depth_label, zorder = 1)
    ax[method].set_yticklabels([])
    make_U_axis(ax[method])

    ax[method].set_xlim(depth_min, depth_max)

add_text(ax_sqz, r"{\bf (a)}")
add_text(ax[OAT], r"{\bf (b)} OAT")
add_text(ax[NOAT], f"{{\\bf (c)}} {NOAT}")

ax_sqz.legend(loc = "best", framealpha = 1)
cbar = fig.colorbar(mesh, cax = cax, label = time_label + " (sec)", format = "%.1f")
cbar_ticks = [ int(tick) for tick in cbar.get_ticks() if tick == int(tick) ]
cbar.set_ticks(cbar_ticks)
cbar.set_ticklabels(cbar_ticks)

# clean up figure
ax_sqz.set_xlim(10, 35)
ax_sqz.set_ylim(sizes[0], sizes[-1])
ax_sqz.set_xlabel(sqz_label, zorder = 1)
ax_sqz.set_ylabel(size_label, zorder = 1)

plt.tight_layout()
plt.gca().set_rasterization_zorder(1)
plt.savefig(fig_dir + "optimization/t_opt_L_2D.pdf",
            rasterized = True, dpi = dpi)
plt.close()

##########################################################################################
# squeezing with decoherence
##########################################################################################

def get_sqz_floor(lattice_depth, lattice_size):

    file_name = data_dir + f"trunc/{NOAT}_{lattice_depth}_{lattice_size}.txt"
    spin_num = int(lattice_size)**2

    correlators = {}
    # if we have previously computed the squeezing floor, use it
    sqz_floor = None
    with open(file_name, "r") as f:
        for line in f:
            if "sqz_floor" in line:
                sqz_floor = float(line.split()[-1])

    if sqz_floor != None: return sqz_floor

    # otherwise, calculate the squeezing floor
    derivs = np.loadtxt(file_name, dtype = complex)
    order_cap = derivs.shape[1]

    times = np.linspace(0, 2, 100) * spin_num**(-2/3)
    times_k = np.array([ times**kk for kk in range(order_cap) ])

    # find squeezing floor at all (reasonable) orders
    orders = np.arange(order_cap//2,order_cap+1)
    sqz_floors = np.zeros(orders.size)
    for order_idx, order in enumerate(orders):
        for op_idx, sqz_op in enumerate(squeezing_ops):
            correlators[sqz_op] = derivs[op_idx,:order] @ times_k[:order,:]

        sqz = squeezing_from_correlators(spin_num, correlators, in_dB = True)

        # get index of first local maximum and inflection point
        d_sqz = sqz[1:] - sqz[:-1] # first derivative
        d_sqz_idx = np.argmax(d_sqz < 0) # index of first local maximum
        dd_sqz = d_sqz[1:] - d_sqz[:-1] # second derivative
        dd_sqz_idx = np.argmax(dd_sqz > 0) # index of first inflection point

        # if either index is zero, it means we did not actually find what we needed
        if  d_sqz_idx == 0:  d_sqz_idx = sqz.size
        if dd_sqz_idx == 0: dd_sqz_idx = sqz.size

        # the peak is at whichever event occurred first
        peak_idx = min(d_sqz_idx, dd_sqz_idx+1)

        # if we have no nans before the peak, use the squeezing value at the peak
        # otherwise, use the maximum pre-nan squeezing value
        if not np.isnan(sqz[:peak_idx+1]).any():
            sqz_floors[order_idx] = sqz[peak_idx]
        else:
            nan_idx = np.argmax(np.isnan(sqz))
            sqz_floors[order_idx] = sqz[:nan_idx].max()

    # save squeezing floor to data file
    sqz_floor = sqz_floors.max()
    with open(file_name, "a") as f:
        f.write(f"# sqz_floor: {sqz_floor}\n")

    return sqz_floor

figsize = (6,3)

# get squeezing data
sqz = {}
sqz[NOAT] = np.zeros((depths_dec.size,sizes_dec.size))
for idx, _ in np.ndenumerate(sqz[NOAT]):
    sqz[NOAT][idx] = get_sqz_floor(depths_dec[idx[0]], sizes_dec[idx[1]])

depths = { NOAT : depths_dec }
sizes = { NOAT : sizes_dec }

sqz[OAT] = -to_dB(pd_read_2D(data_dir + f"optimization/sqz_opt_OAT_dec_L_2D.txt"))
depths[OAT], sizes[OAT], sqz[OAT] \
    = sqz[OAT].index, sqz[OAT].columns, sqz[OAT].values

for method in sqz_methods:
    err = abs(depths[method][1] - depths[method][0]) / 2
    use_depths = (depths[method] >= depth_min-err) & (depths[method] <= depth_max+err)
    depths[method], sqz[method] = depths[method][use_depths], sqz[method][use_depths,:]

sqz_min = min([ sqz[method].min() for method in sqz_methods ])
sqz_max = max([ sqz[method].max() for method in sqz_methods ])

# set up figure panels
ax = {}
fig, ( ax[OAT], ax[NOAT], cax ) \
    = plt.subplots(figsize = figsize, ncols = 3,
                   gridspec_kw = { "width_ratios" : [1,1,0.05] })

# plot data
for method in sqz_methods:
    mesh = ax[method].pcolormesh(depths[method], sizes[method], sqz[method].T,
                                 cmap = plt.get_cmap("jet"),
                                 vmin = sqz_min, vmax = sqz_max,
                                 zorder = 0, rasterized = True)
    ax[method].set_xlabel(depth_label, zorder = 1)
    make_U_axis(ax[method])

    ax[method].set_xlim(depth_min, depth_max)

add_text(ax[OAT], r"{\bf (a)} OAT")
add_text(ax[NOAT], f"{{\\bf (b)}} {NOAT}")

ax[OAT].set_ylabel(size_label)
ax[NOAT].set_yticklabels([])
fig.colorbar(mesh, cax = cax, label = sqz_label)

plt.tight_layout()
plt.gca().set_rasterization_zorder(1)
plt.savefig(fig_dir + "optimization/sqz_opt_dec_L_2D.pdf",
            rasterized = True, dpi = dpi)
plt.close()


import os
import numpy as np
import matplotlib.pyplot as plt

from __impvisu__ import *
import commonFunc

class ShowTemporalGraph:
	def compute(self):
		av_data=[]
		know_id=[]

		for i in self.x_cut:
			n_data,know_id=OpDict.getNewShappedArray(3,self.data[i][self.ekey],know_id)
			av_data.append(
				np.average(
					np.ma.masked_where(
						n_data==0,
						n_data
					)
				)
			)
		return (av_data,know_id) if self.log else (np.log10(av_data),know_id)

	def chooseCustomParam(self,):
		self.log=commonFunc.handlePrintOfCustomParam(
			(
				'\n0\t->\tUse log10 on values',
				'\n1\t->\tDo not use log10 on values'
			),
			2
		)

	#Plot the data depending on the time
	def timeplot(self,val,title,idxtab):
		plt.scatter(self.x_cut,val)
		plt.plot(self.x_cut,val)
		#plt.ylim(0,15)
		plt.title(title+' '+' '.join(str(idxtab)))
		plt.show()

	def chooseWhichParam(self,):
		self.ekey=OpDict.moveIntoDict('Choose element to plot:',self.data,subdict=True)
		self.x1=OpDict.moveIntoDict('Choose range of time to plot the data\nChoose time between the following values:',self.data)
		self.x2=OpDict.moveIntoDict('Choose time between the following values:',self.data)
		self.x_cut = [i for i in self.data.keys() if (float(self.x1)<=float(i)<=float(self.x2))]
		self.chooseCustomParam()

	def expData(self,data):
		self.data=data
		self.chooseWhichParam()
		return self.data,self.x_cut,self.ekey,self.log

	#Writting (time by plot)
	def run(self, data):
		self.data=data
		self.chooseWhichParam()
		val,idxtab=self.compute()
		self.timeplot(val,self.ekey,idxtab)

def tempPlotOneFile(func):
	data=func.getOnlyFile()
	if data==None:
		print('Error with data, you need to reload a dataset')
		return
	ShowTemporalGraph().run(data)

# def main():
# 	ShowTemporalGraph().run(sys.argv[1])

# if __name__ == '__main__':
# 	main()


from itertools import islice
import pytest
from pytest import param as _
from pytest_lazyfixture import lazy_fixture
from blk.types import CycleError

def test_section_with_cycle_sample(dict_section_with_cycle, expected_names):
    names_it = islice(dict_section_with_cycle.names(), 0, 5)
    names = tuple(map(str, names_it))
    assert names == expected_names

@pytest.mark.parametrize(['dict_section', 'has_cycle'], [
    _(lazy_fixture('dict_section_with_cycle'), True, id='cycle'),
    _(lazy_fixture('dict_section_with_same_id_sub'), False, id='same id sub'),
    _(lazy_fixture('dict_section_with_cycle_deep'), True, id='cycle deep'),
    _(lazy_fixture('dict_section_with_same_id_sub_deep'), False, id='same id sub deep'),
])
def test_cycle(dict_section, has_cycle):
    if has_cycle:
        with pytest.raises(CycleError):
            dict_section.check_cycle()
    else:
        dict_section.check_cycle()


import tensorflow as tf
from tensorflow.contrib.layers.python.layers import batch_norm
import functools
import numpy as np
import math

def log_sum_exp(x, axis=1):
    m = tf.reduce_max(x, keep_dims=True)
    return m + tf.log(tf.reduce_sum(tf.exp(x - m), axis=axis))

def lrelu(x, alpha=0.2, name="LeakyReLU"):
    with tf.variable_scope(name):
        return tf.maximum(x , alpha*x) * tf.sqrt(2.0)

def get_weight(shape, gain=1, use_wscale=True, lrmul=1, weight_var='weight'):

    fan_in = np.prod(shape[:-1]) # [kernel, kernel, fmaps_in, fmaps_out] or [in, out]d
    he_std = gain / math.sqrt(int(fan_in)) # He init
    # Equalized learning rate and custom learning rate multiplier.
    if use_wscale:
        init_std = 1.0 / lrmul
        runtime_coef = he_std * lrmul
    else:
        init_std = he_std / lrmul
        runtime_coef = lrmul

    # Create variable.
    init = tf.initializers.random_normal(0, init_std)
    return tf.get_variable(weight_var, shape=shape, initializer=init) * runtime_coef

def conv2d(input_, output_dim, k=4, s=2, gain=1, use_wscale=True, lrmul=1,
           weight_var='w', padding='SAME', scope="conv2d", use_bias=True):

    assert padding in ['SAME', 'VALID', 'REFLECT']
    with tf.variable_scope(scope):
        w = get_weight([k, k, input_.get_shape()[-1], output_dim], gain=gain, use_wscale=use_wscale, lrmul=lrmul,
                       weight_var=weight_var)

        if padding == 'REFLECT':
            input_ = tf.pad(input_, paddings=tf.constant([[0,0], [1,1], [1,1],[0,0]]), mode='REFLECT')
            conv = tf.nn.conv2d(input_, w, strides=[1, s, s, 1], padding='VALID')
        else:
            conv = tf.nn.conv2d(input_, w, strides=[1, s, s, 1], padding=padding)

        if use_bias:
            biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0))
            conv = tf.reshape(tf.nn.bias_add(conv, biases), tf.shape(conv))

        return conv

def fully_connect(input_, output_dim, scope=None, use_sp=False, gain=1, use_wscale=True, lrmul=1, weight_war='w',
                  bias_start=0.0, with_w=False):

  shape = input_.get_shape().as_list()
  with tf.variable_scope(scope or "Linear"):
    w = get_weight([shape[1], output_dim], gain=gain, use_wscale=use_wscale, lrmul=lrmul, weight_var=weight_war)
    bias = tf.get_variable("bias", [output_dim], tf.float32,
      initializer=tf.constant_initializer(bias_start))
    if use_sp:
        mul = tf.matmul(input_, w)
    else:
        mul = tf.matmul(input_, w)
    if with_w:
        return mul + bias, w, bias
    else:
        return mul + bias

# def instance_norm(input, scope="instance_norm", affine=True):
#     with tf.variable_scope(scope):
#         depth = input.get_shape()[-1]
#
#         mean, variance = tf.nn.moments(input, axes=[1, 2], keep_dims=True)
#         epsilon = 1e-5
#         inv = tf.rsqrt(variance + epsilon)
#         normalized = (input - mean) * inv
#         if affine:
#             scale = tf.get_variable("scale", [depth],
#                                     initializer=tf.random_normal_initializer(1.0, 0.02, dtype=tf.float32))
#             offset = tf.get_variable("offset", [depth], initializer=tf.constant_initializer(0.0))
#             return scale * normalized + offset
#         else:
#             return normalized

def instance_norm(x, scope='instance_norm'):
    return tf.contrib.layers.instance_norm(x,
                                           epsilon=1e-05,
                                           center=True, scale=True,
                                           scope=scope)

def Adaptive_instance_norm(input, beta, gamma, epsilon=1e-5, scope="adaptive_instance_norm"):

    ch = beta.get_shape().as_list()[-1]
    with tf.variable_scope(scope):

        mean, variance = tf.nn.moments(input, axes=[1,2], keep_dims=True)
        inv = tf.rsqrt(variance + epsilon)
        normalized = (input - mean) * inv
        beta = tf.reshape(beta, shape=[-1, 1, 1, ch])
        gamma = tf.reshape(gamma, shape=[-1, 1, 1, ch])

        return gamma * normalized + beta

# def modulated_conv2d_layer(input_, style_code, k=3, output_dim=512, padding='SAME', scope='modulated_conv2d'):
#     assert k >= 1 and k % 2 == 1
#     ful = functools.partial(fully_connect)
#     with tf.variable_scope(scope):
#         print(input_)
#         input_ = tf.transpose(input_, [0, 3, 1, 2])
#         w = tf.get_variable('w', [k, k, input_.get_shape()[1], output_dim],
#                             initializer=tf.contrib.layers.variance_scaling_initializer())
#         ww = w[np.newaxis]
#         fmaps = input_.shape[1].value
#         #Modulate
#         style = ful(style_code, output_dim=fmaps)
#         style = style + 1
#         ww = ww * tf.cast(style[:, np.newaxis, np.newaxis, :, np.newaxis], w.dtype)
#         # Demodulate
#         d = tf.rsqrt(tf.reduce_sum(tf.square(ww), axis=[1,2,3]) + 1e-8)
#         ww *= d[:, np.newaxis, np.newaxis, np.newaxis, :]
#         ## Reshape/scale output.
#         input_ = tf.reshape(input_, [1, -1, input_.shape[2], input_.shape[3]])
#         w = tf.reshape(tf.transpose(ww, [1, 2, 3, 0, 4]), [ww.shape[1], ww.shape[2], ww.shape[3], -1])
#
#         input_ = tf.nn.conv2d(input_, w, strides=[1, 1, 1, 1], data_format='NCHW', padding=padding)
#         print(input_)
#         # Reshape/scale output
#         input_ = tf.reshape(input_, [-1, output_dim, input_.shape[2], input_.shape[3]])
#         input_ = tf.transpose(input_, [0, 2, 3, 1])
#
#         return input_

def modulated_conv2d_layer(input_, style_code, k=1, output_dim=512, us=False, gain=1, use_wscale=True, lrmul=1, weight_war='w',
                           padding='SAME', scope='modulated_conv2d'):
    assert k >= 1 and k % 2 == 1
    ful = functools.partial(fully_connect)
    with tf.variable_scope(scope):

        # input_ = tf.transpose(input_, [0, 3, 1, 2])
        # w = tf.get_variable('w', [k, k, input_.get_shape()[-1], output_dim],
        #                     initializer=tf.contrib.layers.variance_scaling_initializer())
        w = get_weight([k, k, input_.get_shape()[-1], output_dim], gain=gain, use_wscale=use_wscale, lrmul=lrmul, weight_var=weight_war)
        ww = w[np.newaxis]
        fmaps = input_.shape[-1].value
        #Modulate
        style = ful(style_code, output_dim=fmaps)
        style = style + 1
        ww = ww * tf.cast(style[:, np.newaxis, np.newaxis, :, np.newaxis], w.dtype)
        # Demodulate
        d = tf.rsqrt(tf.reduce_sum(tf.square(ww), axis=[1,2,3]) + 1e-8)
        ww *= d[:, np.newaxis, np.newaxis, np.newaxis, :]

        ## Reshape/scale output.
        input_ = tf.transpose(input_, [1, 2, 0, 3])
        input_ = tf.reshape(input_, [1, input_.shape[0], input_.shape[1], -1])
        w = tf.reshape(tf.transpose(ww, [1, 2, 3, 0, 4]), [ww.shape[1], ww.shape[2], ww.shape[3], -1])
        if us:
            # print("hha", w.shape)
            # w = tf.transpose(w, [0, 1, 3, 2])
            # print(w.shape)
            # print(input_.shape)
            # input_ = tf.nn.conv2d_transpose(input_, w, output_shape=[input_.shape[0], input_.shape[1]*2,
            #     input_.shape[2]*2, w.shape[-2]], strides=[1, 2, 2, 1])
            # print(input_.shape)
            input_ = upscale(input_, scale=2)
            input_ = tf.nn.conv2d(input_, w, strides=[1, 1, 1, 1], data_format='NHWC', padding=padding)
        else:
            input_ = tf.nn.conv2d(input_, w, strides=[1, 1, 1, 1], data_format='NHWC', padding=padding)
        # Reshape/scale output
        input_ = tf.reshape(input_, [input_.shape[1], input_.shape[2], -1, output_dim])

        input_ = tf.transpose(input_, [2, 0, 1, 3])

        return input_

def Resblock_Mo_Affline_layers(x_init, o_dim, style_code, noise_strength, us=True, scope='resblock'):

    _, x_init_h, x_init_w, input_ch = x_init.get_shape().as_list()
    with tf.variable_scope(scope):

        def shortcut(x):
            if us:
                x = upscale(x, scale=2)
            if input_ch != o_dim:
                x = conv2d(x, output_dim=o_dim, k=1, s=1, scope='conv', use_bias=False)
            return x

        with tf.variable_scope('res1'):

            x = lrelu(x_init)
            # if us:
            #     x = upscale(x, scale=2)
            x = modulated_conv2d_layer(x, style_code, us=us, output_dim=o_dim, scope='mc1')
            noise = tf.random_normal([tf.shape(x)[0], 1, x.shape[2], x.shape[3]], dtype=x.dtype)
            x += noise * tf.cast(noise_strength, x.dtype)

        with tf.variable_scope('res2'):

            x = lrelu(x)
            x = modulated_conv2d_layer(x, style_code, us=False, output_dim=o_dim, scope='mc2')
            noise = tf.random_normal([tf.shape(x)[0], 1, x.shape[2], x.shape[3]], dtype=x.dtype)
            x += noise * tf.cast(noise_strength, x.dtype)

        if o_dim != input_ch or us:
            x_init = shortcut(x_init)

        return (x + x_init) / tf.sqrt(2.0)

def Resblock_AdaIn_Affline_layers(x_init, o_dim, style_code, us=True, scope='resblock'):

    input_ch = x_init.get_shape().as_list()[-1]
    affline_layers = functools.partial(fully_connect, output_dim=input_ch*2)
    affline_layers2 = functools.partial(fully_connect, output_dim=o_dim*2)

    with tf.variable_scope(scope):

        def shortcut(x):
            if us:
                x = upscale(x, scale=2)
            if input_ch != o_dim:
                x = conv2d(x, output_dim=o_dim, k=1, s=1, scope='conv', padding='VALID', use_bias=False)
            return x

        with tf.variable_scope('res1'):
            bg = affline_layers(style_code, scope='fc1')
            beta, gamma = bg[:, 0:input_ch], bg[:, input_ch: input_ch*2]
            x = Adaptive_instance_norm(x_init, beta=beta, gamma=gamma, scope='AdaIn1')
            x = lrelu(x)
            if us:
                x = upscale(x, scale=2)
            x = conv2d(x, o_dim, k=3, s=1, padding='SAME')

        with tf.variable_scope('res2'):
            bg = affline_layers2(style_code, scope='fc2')
            beta, gamma = bg[:, 0:o_dim], bg[:, o_dim: o_dim*2]
            x = Adaptive_instance_norm(x, beta=beta, gamma=gamma, scope='AdaIn2')
            x = lrelu(x)
            x = conv2d(x, o_dim, k=3, s=1, padding='SAME')

        if o_dim != input_ch or us:
            x_init = shortcut(x_init)

        return (x + x_init) / tf.sqrt(2.0)

def Resblock(x_init, o_dim=256, relu_type="lrelu", padding='REFLECT', use_IN=True, ds=True, scope='resblock'):

    dim = x_init.get_shape().as_list()[-1]
    conv1 = functools.partial(conv2d, output_dim=dim, padding=padding, k=3, s=1)
    conv2 = functools.partial(conv2d, output_dim=o_dim, padding=padding, k=3, s=1)
    In = functools.partial(instance_norm)

    input_ch = x_init.get_shape().as_list()[-1]
    with tf.variable_scope(scope):

        def relu(relu_type):
            relu_dict = {
                "relu": tf.nn.relu,
                "lrelu": lrelu
            }
            return relu_dict[relu_type]

        def shortcut(x):
            if input_ch != o_dim:
                x = conv2d(x, output_dim=o_dim, k=1, s=1, scope='conv', use_bias=False)
            if ds:
                x = avgpool2d(x, k=2)
            return x

        if use_IN:
            x = conv1(relu(relu_type)(In(x_init, scope='bn1')), scope='c1')
            if ds:
                x = avgpool2d(x, k=2)
            x = conv2(relu(relu_type)(In(x, scope='bn2')), scope='c2')
        else:
            x = conv1(relu(relu_type)(x_init), scope='c1')
            if ds:
                x = avgpool2d(x, k=2)
            x = conv2(relu(relu_type)(x), scope='c2')

        if input_ch != o_dim or ds:
            x_init = shortcut(x_init)

        return (x + x_init) / tf.sqrt(2.0)  #unit variance

def de_conv(input_, output_dim, k_h=4, k_w=4, d_h=2, d_w=2, scope="deconv2d", with_w=False):

    with tf.variable_scope(scope):

        w = tf.get_variable('w', [k_h, k_w, output_dim[-1], input_.get_shape()[-1]], dtype=tf.float32,
                            initializer=tf.contrib.layers.variance_scaling_initializer())

        deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_dim,
                                        strides=[1, d_h, d_w, 1])

        biases = tf.get_variable('biases', [output_dim[-1]], tf.float32, initializer=tf.constant_initializer(0.0))
        deconv = tf.reshape(tf.nn.bias_add(deconv, biases), deconv.get_shape())

        if with_w:
            return deconv, w, biases
        else:
            return deconv

def avgpool2d(x, k=2):
    return tf.nn.avg_pool(x, ksize=[1, k, k ,1], strides=[1, k, k, 1], padding='SAME')

def Adaptive_pool2d(x, output_size=1):
    input_size = get_conv_shape(x)[-1]
    stride = int(input_size / (output_size))
    kernel_size = input_size - (output_size - 1) * stride
    return tf.nn.avg_pool(x, ksize=[1, kernel_size, kernel_size, 1], strides=[1, kernel_size, kernel_size, 1], padding='SAME')

def upscale(x, scale, method='bilinear'):
    _, h, w, _ = get_conv_shape(x)
    return tf.image.resize(x, size=(h * scale, w * scale), method=method)

def get_conv_shape(tensor):
    shape = int_shape(tensor)
    return shape

def int_shape(tensor):
    shape = tensor.get_shape().as_list()
    return [num if num is not None else -1 for num in shape]

def resize_nearest_neighbor(x, new_size):
    x = tf.image.resize_nearest_neighbor(x, new_size)
    return x

def conv_cond_concat(x, y):
    """Concatenate conditioning vector on feature map axis."""
    x_shapes = x.get_shape()
    y_shapes = y.get_shape()
    y_reshaped = tf.reshape(y, [y_shapes[0], 1, 1, y_shapes[-1]])
    return tf.concat([x , y_reshaped*tf.ones([x_shapes[0], x_shapes[1], x_shapes[2] , y_shapes[-1]])], 3)

def batch_normal(input, scope="scope", reuse=False):
    return batch_norm(input, epsilon=1e-5, decay=0.9, scale=True, scope=scope, reuse=reuse, fused=True, updates_collections=None)

def _l2normalize(v, eps=1e-12):
  return v / (tf.reduce_sum(v ** 2) ** 0.5 + eps)

def getWeight_Decay(scope='discriminator'):
    return tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES, scope=scope))

def getTrainVariable(vars, scope='discriminator'):
    return [var for var in vars if scope in var.name]


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# File: load-resnet.py
# Author: Eric Yujia Huang <yujiah1@andrew.cmu.edu>
#         Yuxin Wu

import argparse
import functools
import numpy as np
import re
import cv2
import six
import tensorflow as tf

from tensorpack import *
from tensorpack.dataflow.dataset import ILSVRCMeta
from tensorpack.utils import logger

from imagenet_utils import ImageNetModel, eval_classification, get_imagenet_dataflow
from resnet_model import resnet_bottleneck, resnet_group

DEPTH = None
CFG = {
    50: ([3, 4, 6, 3]),
    101: ([3, 4, 23, 3]),
    152: ([3, 8, 36, 3])
}

class Model(ModelDesc):
    def inputs(self):
        return [tf.TensorSpec([None, 224, 224, 3], tf.float32, 'input'),
                tf.TensorSpec([None], tf.int32, 'label')]

    def build_graph(self, image, label):
        blocks = CFG[DEPTH]

        bottleneck = functools.partial(resnet_bottleneck, stride_first=True)

        # tensorflow with padding=SAME will by default pad [2,3] here.
        # but caffe conv with stride will pad [3,2]
        image = tf.pad(image, [[0, 0], [3, 2], [3, 2], [0, 0]])
        image = tf.transpose(image, [0, 3, 1, 2])
        with argscope([Conv2D, MaxPooling, GlobalAvgPooling, BatchNorm],
                      data_format='channels_first'), \
                argscope(Conv2D, use_bias=False):
            logits = (LinearWrap(image)
                      .Conv2D('conv0', 64, 7, strides=2, activation=BNReLU, padding='VALID')
                      .MaxPooling('pool0', 3, strides=2, padding='SAME')
                      .apply2(resnet_group, 'group0', bottleneck, 64, blocks[0], 1)
                      .apply2(resnet_group, 'group1', bottleneck, 128, blocks[1], 2)
                      .apply2(resnet_group, 'group2', bottleneck, 256, blocks[2], 2)
                      .apply2(resnet_group, 'group3', bottleneck, 512, blocks[3], 2)
                      .GlobalAvgPooling('gap')
                      .FullyConnected('linear', 1000)())
        tf.nn.softmax(logits, name='prob')
        ImageNetModel.compute_loss_and_error(logits, label)

def get_inference_augmentor():
    # load ResNet mean from Kaiming:
    # from tensorpack.utils.loadcaffe import get_caffe_pb
    # obj = get_caffe_pb().BlobProto()
    # obj.ParseFromString(open('ResNet_mean.binaryproto').read())
    # pp_mean_224 = np.array(obj.data).reshape(3, 224, 224).transpose(1,2,0)

    meta = ILSVRCMeta()
    pp_mean = meta.get_per_pixel_mean()
    pp_mean_224 = pp_mean[16:-16, 16:-16, :]

    transformers = [
        imgaug.ResizeShortestEdge(256),
        imgaug.CenterCrop((224, 224)),
        imgaug.MapImage(lambda x: x - pp_mean_224),
    ]
    return transformers

def run_test(params, input):
    pred_config = PredictConfig(
        model=Model(),
        session_init=SmartInit(params),
        input_names=['input'],
        output_names=['prob']
    )
    predict_func = OfflinePredictor(pred_config)

    prepro = imgaug.AugmentorList(get_inference_augmentor())
    im = cv2.imread(input).astype('float32')
    im = prepro.augment(im)
    im = np.reshape(im, (1, 224, 224, 3))
    outputs = predict_func(im)
    prob = outputs[0]

    ret = prob[0].argsort()[-10:][::-1]
    print(ret)
    meta = ILSVRCMeta().get_synset_words_1000()
    print([meta[k] for k in ret])

def name_conversion(caffe_layer_name):
    """ Convert a caffe parameter name to a tensorflow parameter name as
        defined in the above model """
    # beginning & end mapping
    NAME_MAP = {'bn_conv1/beta': 'conv0/bn/beta',
                'bn_conv1/gamma': 'conv0/bn/gamma',
                'bn_conv1/mean/EMA': 'conv0/bn/mean/EMA',
                'bn_conv1/variance/EMA': 'conv0/bn/variance/EMA',
                'conv1/W': 'conv0/W', 'conv1/b': 'conv0/b',
                'fc1000/W': 'linear/W', 'fc1000/b': 'linear/b'}
    if caffe_layer_name in NAME_MAP:
        return NAME_MAP[caffe_layer_name]

    s = re.search('([a-z]+)([0-9]+)([a-z]+)_', caffe_layer_name)
    if s is None:
        s = re.search('([a-z]+)([0-9]+)([a-z]+)([0-9]+)_', caffe_layer_name)
        layer_block_part1 = s.group(3)
        layer_block_part2 = s.group(4)
        assert layer_block_part1 in ['a', 'b']
        layer_block = 0 if layer_block_part1 == 'a' else int(layer_block_part2)
    else:
        layer_block = ord(s.group(3)) - ord('a')
    layer_type = s.group(1)
    layer_group = s.group(2)

    layer_branch = int(re.search('_branch([0-9])', caffe_layer_name).group(1))
    assert layer_branch in [1, 2]
    if layer_branch == 2:
        layer_id = re.search('_branch[0-9]([a-z])/', caffe_layer_name).group(1)
        layer_id = ord(layer_id) - ord('a') + 1

    TYPE_DICT = {'res': 'conv{}', 'bn': 'conv{}/bn'}
    layer_type = TYPE_DICT[layer_type].format(layer_id if layer_branch == 2 else 'shortcut')

    tf_name = caffe_layer_name[caffe_layer_name.index('/'):]
    tf_name = 'group{}/block{}/{}'.format(
        int(layer_group) - 2, layer_block, layer_type) + tf_name
    return tf_name

def convert_param_name(param):
    resnet_param = {}
    for k, v in six.iteritems(param):
        try:
            newname = name_conversion(k)
        except Exception:
            logger.error("Exception when processing caffe layer {}".format(k))
            raise
        logger.info("Name Transform: " + k + ' --> ' + newname)
        resnet_param[newname] = v
    return resnet_param

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--load', required=True,
                        help='.npz model file generated by tensorpack.utils.loadcaffe')
    parser.add_argument('-d', '--depth', help='resnet depth', required=True, type=int, choices=[50, 101, 152])
    parser.add_argument('--input', help='an input image')
    parser.add_argument('--convert', help='npz output file to save the converted model')
    parser.add_argument('--eval', help='ILSVRC dir to run validation on')

    args = parser.parse_args()
    DEPTH = args.depth

    param = dict(np.load(args.load))
    param = convert_param_name(param)

    if args.convert:
        assert args.convert.endswith('.npz')
        np.savez_compressed(args.convert, **param)

    if args.eval:
        ds = get_imagenet_dataflow(args.eval, 'val', 128, get_inference_augmentor())
        eval_classification(Model(), SmartRestore(param), ds)
    elif args.input:
        run_test(param, args.input)


String=input("Enter A String:")
firstChar=String[0]
#dummy variable for answer
result=firstChar
# Iterating from 1 index
for i in range(1,len(String)):
    if(String[i]==firstChar):
        result=result+"$"
    else:
        result=result+String[i]
print(result)


import os
import time
import logging
import pickle
import multiprocessing
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor

import numpy as np
import torch
import torch.nn.functional as F
from PIL import Image
import imagehash
import n2

from clip_search.config import FEATURE_DIM, INDEX_DIR, PATH_DIR, HASH_DIR, EMBEDDING_DIR, ADDR_IMAGE, PORT_IMAGE

def measure_elapsed_time(func, func_str, ms=True):
    start_time = time.time()
    ret = func()
    end_time = time.time()
    elapsed_time = end_time - start_time
    if ms == True:
        logging.info(f"{func_str} elapsed_time: {elapsed_time*1000:.2f}ms")
    else:
        logging.info(f"{func_str} elapsed_time: {elapsed_time:.2f}s")
    return ret

# warm up for HNSW search
def warm_up_search(index, warm_up_count):
    if index is None or warm_up_count == 0:
        return

    for i in range(warm_up_count):
        query = F.normalize(torch.rand(1, FEATURE_DIM), p=2, dim=-1).squeeze(0).tolist()
        index.search_by_vector(query, 100, ef_search=300, include_distances=False)
    logging.info(f"[{os.getpid()}] warmup finish.")

# Flask app serving with wsgi
def wsgi_serve(app, wsgi, host, port, num_workers=1, num_threads=1, search_server=False, index=None, warm_up=0):
    if search_server == True:
        assert index is not None

    # using waitress
    if wsgi == "waitress":
        if search_server == True:
            warm_up_search(index, warm_up)
        import waitress
        waitress.serve(app, host=host, port=port, threads=num_threads, connection_limit=65536, asyncore_use_poll=True)

    # using gunicorn
    elif wsgi == "gunicorn":
        from gunicorn.app.base import BaseApplication
        from gunicorn.six import iteritems
        options = {
            'bind': f"{host}:{port}",
            'workers': num_workers,
            'threads': num_threads,
        }
        if search_server == True:
            options['post_fork'] = lambda server, worker: warm_up_search(index, warm_up)

        class StandaloneApplication(BaseApplication):
            def __init__(self, app, options=None):
                self.options = options or {}
                self.application = app
                super(StandaloneApplication, self).__init__()

            def load_config(self):
                config = dict([(key, value) for key, value in iteritems(self.options)
                               if key in self.cfg.settings and value is not None])
                for key, value in iteritems(config):
                    self.cfg.set(key.lower(), value)

            def load(self):
                return self.application

        StandaloneApplication(app, options).run()
    else:
        logging.error("wsgi argument error. It should be in ['gunicorn', 'waitress'].")

# save HNSW n2 index as file
def save_index(index, index_num):
    os.makedirs(INDEX_DIR, exist_ok=True)
    index.save(os.path.join(INDEX_DIR, f"{index_num:03d}.pt"))

# load HNSW n2 index from file
def load_index(index_num, mmap=True):
    index = n2.HnswIndex(FEATURE_DIM, metric='dot')
    index.load(os.path.join(INDEX_DIR, f"{index_num:03d}.pt"), use_mmap=mmap)
    return index

# save path list as file
def save_path_list(path_list, index_num):
    os.makedirs(PATH_DIR, exist_ok=True)
    with open(os.path.join(PATH_DIR, f"{index_num:03d}.pkl"), "wb") as f:
        pickle.dump(path_list, f)

# load path list from file
def load_path_list(index_num):
    with open(os.path.join(PATH_DIR, f"{index_num:03d}.pkl"), "rb") as f:
        path_list = pickle.load(f)
    return path_list

# save hash list as file
def save_hash_list(hash_list, index_num):
    os.makedirs(HASH_DIR, exist_ok=True)
    with open(os.path.join(HASH_DIR, f"{index_num:03d}.pkl"), "wb") as f:
        pickle.dump(hash_list, f)

# load hash list from file
def load_hash_list(index_num):
    with open(os.path.join(HASH_DIR, f"{index_num:03d}.pkl"), "rb") as f:
        hash_list = pickle.load(f)
    return hash_list

# load datasets from (many) .pt files
def read_datasets(datasets):
    if type(datasets) == str:
        datasets = [datasets]

    for dataset in datasets:
        logging.info(f"**********{dataset}**********")
        dataset_path = os.path.join(EMBEDDING_DIR, dataset)
        if not os.path.isdir(dataset_path):
            logging.error(f"{dataset_path} doesn't exists!")
            continue

        for i in range(1000):  # read 'EMBEDDING_DIR/dataset/xxx.pt
            fname = f"{i:03d}.pt"
            file_path = os.path.join(dataset_path, fname)

            if os.path.isfile(file_path):
                content = torch.load(file_path, map_location="cpu")
                assert len(content['path']) == content['feature'].shape[0]
                yield content
            else:
                break

# features generator
def generate_features(datasets):
    for content in read_datasets(datasets):
        yield content['embed']

# path_list generator
def generate_path_list(datasets):
    for content in read_datasets(datasets):
        yield content['path']

# path_list generator
def generate_hash_list(datasets):
    for content in read_datasets(datasets):
        yield content['hash']

# convert a relative path to abs path
def convert_path_to_url(path):
    root_path = ""
    dataset, *keys = path.split('/')

    web_prefix = f"{ADDR_IMAGE}:{PORT_IMAGE}/dataset"
    s3_prefix = "https://s3-us-east-2.amazonaws.com"
    if dataset == "CC12M":
        dataset_path = os.path.join(web_prefix, "CC12M")
    elif dataset == "imagenet21k":
        dataset_path = os.path.join(web_prefix, "imagenet21k")
    elif dataset == "openimages":
        dataset_path = os.path.join(web_prefix, "openimages")
    elif dataset == "signals17M":
        dataset_path = os.path.join(web_prefix, "signals17M")
    elif dataset == "my-s3-bucket-name":
        dataset_path = os.path.join(s3_prefix, dataset)
    else:
        logging.error(f"path:{path} is incorrect.")
        return ""

    return path.replace(dataset, dataset_path)

# convert imagehash to integer
def convert_hash_to_integer(hash):
    integer = 0
    for bit in hash.hash.reshape(-1):
        integer = (integer << 1) + int(bit)
    return integer

# remove duplicated images (criterion: imagehash)
def remove_duplicated_images(score_list, path_list, hash_list, server_index_list):
    if len(path_list) == 0:
        return [], [], []

    hash_dict = defaultdict(list)
    for idx, hash in enumerate(hash_list):
        hash_dict[hash].append(idx)
    dup_generator = (index_list[1:] for index_list in hash_dict.values() if len(index_list)>1)

    duplicated_index = []
    [duplicated_index.extend(dup) for dup in dup_generator]

    duplicated_removed_index = np.setdiff1d(np.arange(len(path_list)), np.array(duplicated_index))
    duplicated_removed_score = [score_list[idx] for idx in duplicated_removed_index]
    duplicated_removed_path = [path_list[idx] for idx in duplicated_removed_index]
    duplicated_removed_server_index = [server_index_list[idx] for idx in duplicated_removed_index]
    return duplicated_removed_score, duplicated_removed_path, duplicated_removed_server_index


# move n random files from one directory to another
import os
import random
import shutil

oldPath = 'C:/Users/oldPath/'
newPath = 'C:/Users/newPath/'

def randomFile():
    # get a random file with json ext from the folder
    fileList = os.listdir(oldPath)
    fileName = fileList[random.randint(0, len(fileList) - 1)]
    if fileName.endswith('.json'):
        return fileName
    else:
        return randomFile()

def main():
    # loop for 100 times / files
    n = 100
    for i in range(n):
        # get a random file from the folder
        fileName = randomFile()
        print(fileName)
        # find same fileName with png ext
        for file in os.listdir(oldPath):
            if fileName.replace('.json', '.png') == file:
                print(file)
                # move the file to new folder
                shutil.move(oldPath + fileName, newPath + fileName)
                shutil.move(oldPath + file, newPath + file)
                break

# Driver Code
if __name__ == '__main__':
    main()


from foodgram.settings import CSRF_TRUSTED_ORIGINS

from django.contrib.auth.hashers import make_password
from django.shortcuts import get_object_or_404

from rest_framework import serializers

from drf_extra_fields.fields import Base64ImageField

from recipes.models import Ingredient, Recipe, Tag, RecipeIngredient
from users.models import User, Follow

class NestedRecipeSerializer(serializers.ModelSerializer):
    image = serializers.SerializerMethodField()

    class Meta:
        model = Recipe
        fields = (
            'id',
            'name',
            'image',
            'cooking_time'
        )

    def get_image(self, obj):
        request = self.context['request']
        image_url = obj.image.url
        return request.build_absolute_uri(image_url).replace(
            'http://backend:8000', CSRF_TRUSTED_ORIGINS[0]
        )

class UserSerializer(serializers.ModelSerializer):
    is_subscribed = serializers.SerializerMethodField()

    recipes = NestedRecipeSerializer(many=True, read_only=True)

    class Meta:
        model = User
        fields = (
            'email',
            'id',
            'username',
            'first_name',
            'last_name',
            'is_subscribed',
            'password',
            'recipes'
        )
        extra_kwargs = {
            'password': {'write_only': True}
        }

    def validate_password(self, value):
        return make_password(value)

    def get_is_subscribed(self, obj):
        if self.context['request'].user.is_authenticated:
            return Follow.objects.filter(
                user=self.context['request'].user,
                following=obj
            ).exists()
        return False

class IngredientSerializer(serializers.ModelSerializer):
    class Meta:
        model = Ingredient
        fields = (
            'id',
            'name',
            'measurement_unit',
        )

class NestedIngredientSerializer(serializers.ModelSerializer):
    id = serializers.IntegerField(min_value=1)

    class Meta:
        model = Ingredient
        fields = (
            'id',
            'name',
            'measurement_unit'
        )

class TagSerializer(serializers.ModelSerializer):
    class Meta:
        model = Tag
        fields = (
            'id',
            'name',
            'color',
            'slug',
        )

class RecipeIngredientReadSerializer(serializers.ModelSerializer):
    ingredient = NestedIngredientSerializer()

    class Meta:
        model = RecipeIngredient
        fields = (
            'id',
            'ingredient'
        )

    def to_representation(self, instance):
        ret = super().to_representation(instance)
        del ret['ingredient']
        ret['id'] = instance.ingredient.pk
        ret['name'] = instance.ingredient.name
        ret['measurement_unit'] = instance.ingredient.measurement_unit
        ret['amount'] = instance.amount
        return ret

class RecipeIngredientWriteSerializer(serializers.Serializer):
    id = serializers.IntegerField(min_value=1)
    amount = serializers.IntegerField(min_value=1)

class RecipeFavoritesShoppingCartSerializer(serializers.ModelSerializer):
    class Meta:
        model = Recipe
        fields = (
            'id',
            'name',
            'image',
            'cooking_time'
        )
        read_only_fields = (
            'id',
            'name',
            'image',
            'cooking_time'
        )

class RecipeReadSerializer(serializers.ModelSerializer):
    is_favorited = serializers.SerializerMethodField()

    is_in_shopping_cart = serializers.SerializerMethodField()

    tags = TagSerializer(many=True)
    ingredients = RecipeIngredientReadSerializer(many=True)
    author = UserSerializer(
        read_only=True,
    )
    image = serializers.SerializerMethodField()

    class Meta:
        model = Recipe
        fields = (
            'id',
            'tags',
            'author',
            'ingredients',
            'is_favorited',
            'is_in_shopping_cart',
            'name',
            'image',
            'text',
            'cooking_time',

        )
        depth = 1

    def get_image(self, obj):
        request = self.context['request']
        image_url = obj.image.url
        return request.build_absolute_uri(image_url).replace(
            'http://backend:8000', CSRF_TRUSTED_ORIGINS[0]
        )

    def get_is_favorited(self, obj):
        if self.context['request'].user.is_authenticated:
            return obj in self.context['request'].user.favorite_recipes.all()
        return False

    def get_is_in_shopping_cart(self, obj):
        if self.context['request'].user.is_authenticated:
            return obj in self.context['request'].user.shopping_cart.all()
        return False

class RecipeWriteSerializer(serializers.ModelSerializer):

    is_favorited = serializers.SerializerMethodField()

    is_in_shopping_cart = serializers.SerializerMethodField()

    tags = serializers.PrimaryKeyRelatedField(
        queryset=Tag.objects.all(),
        many=True
    )
    ingredients = RecipeIngredientWriteSerializer(many=True)
    author = UserSerializer(
        read_only=True,
    )
    image = Base64ImageField(
        required=False,
        allow_null=True
    )

    class Meta:
        model = Recipe
        fields = (
            'id',
            'tags',
            'author',
            'ingredients',
            'is_favorited',
            'is_in_shopping_cart',
            'name',
            'image',
            'text',
            'cooking_time',

        )
        depth = 1

    def create(self, validated_data):
        ingredients_data = validated_data.pop('ingredients')
        tags = validated_data.pop('tags')
        recipe = Recipe.objects.create(**validated_data)
        recipe.tags.set(tags)
        objs = []
        for ingredient_data in ingredients_data:
            id = ingredient_data.pop('id')
            amount = ingredient_data.pop('amount')
            obj = RecipeIngredient(
                recipe=recipe,
                ingredient=get_object_or_404(Ingredient, pk=id),
                amount=amount
            )
            objs.append(obj)
        RecipeIngredient.objects.bulk_create(objs)
        return recipe

    def update(self, instance, validated_data):
        if 'ingredients' in validated_data.keys():
            instance.ingredients.all().delete()
            ingredients_data = validated_data.pop('ingredients')
            objs = []
            for ingredient_data in ingredients_data:
                id = ingredient_data.pop('id')
                amount = ingredient_data.pop('amount')
                obj, _ = RecipeIngredient.objects.get_or_create(
                    recipe=instance,
                    ingredient=get_object_or_404(Ingredient, pk=id),
                    amount=amount
                )
                objs.append(obj)
            RecipeIngredient.objects.bulk_create(objs, ignore_conflicts=True)
        tags = validated_data.get('tags', instance.tags)
        instance.tags.set(tags)
        instance.name = validated_data.get('name', instance.name)
        instance.text = validated_data.get('text', instance.text)
        instance.image = validated_data.get('image', instance.image)
        instance.cooking_time = validated_data.get(
            'cooking_time',
            instance.cooking_time
        )
        instance.save()
        return instance

    def get_is_favorited(self, obj):
        if self.context['request'].user.is_authenticated:
            return obj in self.context['request'].user.favorite_recipes.all()
        return False

    def get_is_in_shopping_cart(self, obj):
        if self.context['request'].user.is_authenticated:
            return obj in self.context['request'].user.shopping_cart.all()
        return False


import os
import csv

def process_file(current_directory, file_name, output_directory):

    file = open(current_directory + file_name, 'r', encoding="utf-8")

    header = file.readline().split(',')

    title_index = header.index('title')
    selftext_index = header.index('selftext')
    url_index = header.index('url')

    raw_text = file.read()

    file.close()

    while ",,," in raw_text:
        raw_text = raw_text.replace(",,,", ",0,0,")
    while ",," in raw_text:
        raw_text = raw_text.replace(",,", ",0,")
    while ",\n" in raw_text:
        raw_text = raw_text.replace(",\n", ",0\n")

    a = 0
    b = 0

    database = []
    ls = []

    while True:
        if b == len(raw_text):
            ls.append(raw_text[a: b])

            if len(ls) == len(header):
                database.append(ls)

            break
        elif (raw_text[b] == ',' and raw_text[b-1] == '\"' and raw_text[b-2] != '\"') or (raw_text[b] == ',' and raw_text[a] != '\"'):
            ls.append(raw_text[a: b])
            raw_text = raw_text[(b - a + 1):]
            a = 0
            b = 0
            if len(ls) == len(header) - 1:
                ls.append('0')
                database.append(ls)

                raw_text = raw_text[1:]
                # print(ls)
                ls = []
        elif raw_text[b] == '\"' and raw_text[b - 1] == ',':
            a = b

        b += 1

    new_file = open(output_directory + file_name, 'w', encoding="utf-8")
    new_file.writelines("title,selftext,url\n")
    for line in database:
        s = line[title_index] + ',' + line[selftext_index] + ',' + line[url_index] + '\n'
        new_file.writelines(s)

    new_file.close()

    print(file_name)
    return database

"""
for file_name in os.listdir("reddit-top-2.5-million/data"):
    if file_name not in os.listdir("data"):
        process_file("reddit-top-2.5-million/data/", file_name, "data/")

file_name = "3amjokes.csv"
process_file("reddit-top-2.5-million/data/", file_name, "data/")
"""

def process_file_2(current_directory, file_name, output_directory, columns):
    import csv
    if current_directory[:-1] != '/':
        current_directory += '/'

    csvfile = open(current_directory + file_name, 'r', encoding="utf-8")
csvfile = open("reddit-top-2.5-million/data/3amjokes.csv", 'r', encoding="utf-8")
spamreader = csv.reader(csvfile)