import sys
from dataclasses import dataclass, field
from itertools import repeat
from pathlib import Path
from typing import Optional, Union

from ..common import deep
from ..common.binary_reader import BinaryReader
from ..common.deep import Json
from ..common.gltf import parse_glb
from ..common.logging import get_logger
from .license_validation import validate_license

logger = get_logger(__name__)

@dataclass(frozen=True)
class Vrm0MaterialProperty:
    name: str
    shader: str
    render_queue: Optional[int]
    keyword_map: dict[str, bool]
    tag_map: dict[str, str]
    float_properties: dict[str, float]
    vector_properties: dict[str, list[float]]
    texture_properties: dict[str, int]

    @staticmethod
    def create(json_dict: Json) -> "Vrm0MaterialProperty":
        fallback = Vrm0MaterialProperty(
            name="Undefined",
            shader="VRM_USE_GLTFSHADER",
            render_queue=None,
            keyword_map={},
            tag_map={},
            float_properties={},
            vector_properties={},
            texture_properties={},
        )
        if not isinstance(json_dict, dict):
            return fallback

        name = json_dict.get("name")
        if not isinstance(name, str):
            name = fallback.name

        shader = json_dict.get("shader")
        if not isinstance(shader, str):
            shader = fallback.shader

        render_queue = json_dict.get("renderQueue")
        if not isinstance(render_queue, int):
            render_queue = fallback.render_queue

        raw_keyword_map = json_dict.get("keywordMap")
        if isinstance(raw_keyword_map, dict):
            keyword_map = {
                k: v for k, v in raw_keyword_map.items() if isinstance(v, bool)
            }
        else:
            keyword_map = fallback.keyword_map

        raw_tag_map = json_dict.get("tagMap")
        if isinstance(raw_tag_map, dict):
            tag_map = {k: v for k, v in raw_tag_map.items() if isinstance(v, str)}
        else:
            tag_map = fallback.tag_map

        raw_float_properties = json_dict.get("floatProperties")
        if isinstance(raw_float_properties, dict):
            float_properties = {
                k: v
                for k, v in raw_float_properties.items()
                if isinstance(v, (float, int))
            }
        else:
            float_properties = fallback.float_properties

        raw_vector_properties = json_dict.get("vectorProperties")
        if isinstance(raw_vector_properties, dict):
            vector_properties: dict[str, list[float]] = {}
            for k, v in raw_vector_properties.items():
                if not isinstance(v, list):
                    continue
                float_v: list[float] = []
                ok = True
                for e in v:
                    if not isinstance(e, (float, int)):
                        ok = False
                        break
                    float_v.append(float(e))
                if ok:
                    vector_properties[k] = float_v
        else:
            vector_properties = fallback.vector_properties

        raw_texture_properties = json_dict.get("textureProperties")
        if isinstance(raw_texture_properties, dict):
            texture_properties = {
                k: v for k, v in raw_texture_properties.items() if isinstance(v, int)
            }
        else:
            texture_properties = fallback.texture_properties

        return Vrm0MaterialProperty(
            name=name,
            shader=shader,
            render_queue=render_queue,
            keyword_map=keyword_map,
            tag_map=tag_map,
            float_properties=float_properties,
            vector_properties=vector_properties,
            texture_properties=texture_properties,
        )

@dataclass
class ImageProperties:
    name: str
    filepath: Path
    filetype: str

@dataclass
class ParseResult:
    filepath: Path
    json_dict: dict[str, Json] = field(default_factory=dict)
    spec_version_number: tuple[int, int] = (0, 0)
    spec_version_str: str = "0.0"
    spec_version_is_stable: bool = True
    vrm0_extension: dict[str, Json] = field(init=False, default_factory=dict)
    vrm1_extension: dict[str, Json] = field(init=False, default_factory=dict)
    hips_node_index: Optional[int] = None
    image_properties: list[ImageProperties] = field(init=False, default_factory=list)
    vrm0_material_properties: list[Vrm0MaterialProperty] = field(
        init=False, default_factory=list
    )
    skins_joints_list: list[list[int]] = field(init=False, default_factory=list)
    skins_root_node_list: list[int] = field(init=False, default_factory=list)

def remove_unsafe_path_chars(filename: str) -> str:
    unsafe_chars = {
        0: "\x00",
        1: "\x01",
        2: "\x02",
        3: "\x03",
        4: "\x04",
        5: "\x05",
        6: "\x06",
        7: "\x07",
        8: "\x08",
        9: "\t",
        10: "\n",
        11: "\x0b",
        12: "\x0c",
        13: "\r",
        14: "\x0e",
        15: "\x0f",
        16: "\x10",
        17: "\x11",
        18: "\x12",
        19: "\x13",
        20: "\x14",
        21: "\x15",
        22: "\x16",
        23: "\x17",
        24: "\x18",
        25: "\x19",
        26: "\x1a",
        27: "\x1b",
        28: "\x1c",
        29: "\x1d",
        30: "\x1e",
        31: "\x1f",
        34: '"',
        42: "*",
        47: "/",
        58: ":",
        60: "<",
        62: ">",
        63: "?",
        92: "\\",
        124: "|",
    }  # 32:space #33:!
    remove_table = str.maketrans(
        "".join([chr(unsafe_char) for unsafe_char in unsafe_chars]),
        "".join(repeat("_", len(unsafe_chars))),
    )
    safe_filename = filename.translate(remove_table)
    return safe_filename

#  "accessorの順に" データを読み込んでリストにしたものを返す
def decode_bin(
    json_dict: dict[str, Json], binary: bytes
) -> list[list[Union[int, float, list[int], list[float]]]]:
    br = BinaryReader(binary)
    # This list indexed by accessor index
    decoded_binary: list[list[Union[int, float, list[int], list[float]]]] = []
    buffer_view_dicts = json_dict.get("bufferViews")
    if not isinstance(buffer_view_dicts, list):
        buffer_view_dicts = []
    accessor_dicts = json_dict.get("accessors")
    if not isinstance(accessor_dicts, list):
        return []
    type_num_dict = {"SCALAR": 1, "VEC2": 2, "VEC3": 3, "VEC4": 4, "MAT4": 16}
    for accessor_index, accessor_dict in enumerate(accessor_dicts):
        if not isinstance(accessor_dict, dict):
            continue
        accessor_type = accessor_dict.get("type")
        if not isinstance(accessor_type, str):
            continue
        type_num = type_num_dict.get(accessor_type)
        if not isinstance(type_num, int):
            logger.warning(f"Unrecognized accessor type: {accessor_type}")
            continue
        buffer_view_index = accessor_dict.get("bufferView")
        if not isinstance(buffer_view_index, int):
            logger.warning(
                f"accessors[{accessor_index}] doesn't have bufferView that is not implemented yet"
            )
            decoded_binary.append([])
            continue
        buffer_view_dict = buffer_view_dicts[buffer_view_index]
        if not isinstance(buffer_view_dict, dict):
            continue
        buffer_view_byte_offset = buffer_view_dict.get("byteOffset")
        if not isinstance(buffer_view_byte_offset, int):
            buffer_view_byte_offset = 0
        br.set_pos(buffer_view_byte_offset)
        data_list: list[Union[int, float, list[int], list[float]]] = []
        accessor_count = accessor_dict.get("count")
        if not isinstance(accessor_count, int):
            accessor_count = 0
        component_type = accessor_dict.get("componentType")
        if not isinstance(component_type, int):
            raise ValueError(f"Unsupported component type: {component_type}")
        for _ in range(accessor_count):
            if type_num == 1:
                single_data = br.read_as_data_type(component_type)
                data_list.append(single_data)
            else:
                multiple_data = []
                for _ in range(type_num):
                    multiple_data.append(br.read_as_data_type(component_type))
                data_list.append(multiple_data)
        decoded_binary.append(data_list)

    return decoded_binary

@dataclass
class VrmParser:
    filepath: Path
    extract_textures_into_folder: bool
    make_new_texture_folder: bool
    license_validation: bool
    decoded_binary: list[list[Union[int, float, list[int], list[float]]]] = field(
        init=False, default_factory=list
    )
    json_dict: dict[str, Json] = field(init=False, default_factory=dict)

    def parse(self) -> ParseResult:
        # bin chunkは一つだけであることを期待
        json_dict, _ = parse_glb(self.filepath.read_bytes())
        self.json_dict = json_dict

        if self.license_validation:
            validate_license(self.json_dict)

        parse_result = ParseResult(filepath=self.filepath, json_dict=self.json_dict)
        self.vrm_extension_read(parse_result)
        self.material_read(parse_result)

        return parse_result

    def vrm_extension_read(self, parse_result: ParseResult) -> None:
        vrm1_dict = deep.get(self.json_dict, ["extensions", "VRMC_vrm"])
        if isinstance(vrm1_dict, dict):
            self.vrm1_extension_read(parse_result, vrm1_dict)
            return

        vrm0_dict = deep.get(self.json_dict, ["extensions", "VRM"])
        if isinstance(vrm0_dict, dict):
            self.vrm0_extension_read(parse_result, vrm0_dict)

    def vrm0_extension_read(
        self, parse_result: ParseResult, vrm0_dict: dict[str, Json]
    ) -> None:
        spec_version = vrm0_dict.get("specVersion")
        if isinstance(spec_version, str):
            parse_result.spec_version_str = spec_version
        parse_result.vrm0_extension = vrm0_dict

        human_bones = deep.get(vrm0_dict, ["humanoid", "humanBones"], [])
        if not isinstance(human_bones, list):
            raise ValueError("No human bones")

        hips_node_index: Optional[int] = None
        for human_bone in human_bones:
            if isinstance(human_bone, dict) and human_bone.get("bone") == "hips":
                index = human_bone.get("node")
                if isinstance(index, int):
                    hips_node_index = index

        if not isinstance(hips_node_index, int):
            logger.warning("No hips bone index found")
            return

        parse_result.hips_node_index = hips_node_index

    def vrm1_extension_read(
        self, parse_result: ParseResult, vrm1_dict: dict[str, Json]
    ) -> None:
        parse_result.vrm1_extension = vrm1_dict
        parse_result.spec_version_number = (1, 0)
        parse_result.spec_version_is_stable = False

        hips_node_index = deep.get(
            vrm1_dict, ["humanoid", "humanBones", "hips", "node"]
        )
        if not isinstance(hips_node_index, int):
            logger.warning("No hips bone index found")
            return
        parse_result.hips_node_index = hips_node_index

    # ここからマテリアル
    def material_read(self, parse_result: ParseResult) -> None:
        material_dicts = self.json_dict.get("materials")
        if not isinstance(material_dicts, list):
            return
        for index in range(len(material_dicts)):
            parse_result.vrm0_material_properties.append(
                Vrm0MaterialProperty.create(
                    deep.get(
                        self.json_dict,
                        ["extensions", "VRM", "materialProperties", index],
                    )
                )
            )

if __name__ == "__main__":
    VrmParser(
        Path(sys.argv[1]),
        extract_textures_into_folder=True,
        make_new_texture_folder=True,
        license_validation=True,
    ).parse()


import codecs
import csv
import string
import time
import re

import nltk
from nltk.corpus import stopwords

class TrumpDataReader:

    def __init__(self, training_file, case="-t"):
        self.tweets_training = []
        self.tweets_generating = []
        self.case = case

        self.negating = False

        if training_file:
            self.read_and_process_data(training_file, self.case)

    def read_and_process_data(self, training_file, case):
        """
        :param training_file: file to be read and processed.
        :return: void
        """
        with codecs.open(training_file, 'r', 'utf-8') as f:
            f.readline()  # formatting line
            i = 0
            k = 1000
            start_time = time.time()
            for row in f:
                self.negating = False
                tweet = row.split("§")[0]
                clean_tweet_training = self.process_tweet(tweet, "-t")
                clean_tweet_generating = self.process_tweet(tweet, "-lm")
                if clean_tweet_training != "":
                    self.tweets_training.append(clean_tweet_training)
                    self.tweets_generating.append(clean_tweet_generating)
                    #print(clean_tweet_training)
                    #print(clean_tweet_generating)

                if i % k == 0:
                    end_time = time.time()
                    duration = end_time - start_time
                    print(duration)
                i += 1
            print('Done!')

    def process_tweet(self, tweet, case):
        """
        :param tweet: one tweet read from the training document, only contains the body of the tweet.
        :return: tweet, tweet body with dimensionality reduced.
        """
        # tokens = nltk.word_tokenize(tweet) # Gör att kontrollen för länkar inte fungerar
        tokens = tweet.split()

        if tokens[0] == "RT":
            return ""

        for index in range(len(tokens)):
            tokens[index] = self.process_token(tokens[index], case)

        while "" in tokens:
            tokens.remove("")

        if case == "-t":
            tokens = set(tokens)
        tweet = self.list_to_string(tokens)

        return tweet

    def process_token(self, token, case):
        """
        Method to process a single token in a tweet
        :param token: one word from tweet
        :return: processed token
        """
        if token[0] == "@":
            token = "@user"

        elif token[:4] == "http" or token[:4] == "www." or token[len(token)-4:] == ".com":
            token = "adress.com"

        elif token == "&":
            token = ""

        else:
            if case == "-t":

                token = token.lower().strip()

                stop_words = set(stopwords.words("english"))  # Går det snabbare om man har self.stopwords?

                # if token in string.punctuation:
                #    self.negating = False
                #    return ""

                if any(char in token for char in ".,:;()!?"):
                    self.negating = False

                if token in stop_words:
                    return ""

                if token[len(token)-3:] == "n't":
                    self.negating = True

                # if token == "n't":
                #    self.negating = True
                #    return ""
                token = re.sub(r'\s[^\s\w]+\s', ' ', token)
                token = re.sub(r'\d+\s?|\n|[^\s\w]', '', token)

                if self.negating:
                    token = "NOT_" + token

            elif case == "-lm":
                pass

        return token

    def list_to_string(self, split_tweet):
        """
        :param split_tweet: tweet split into list
        :return: split_tweet joined to a string
        """
        return " ".join(split_tweet)

def main():
    TrumpDataReader("Data/tweet_data.txt")

if __name__ == "__main__":
    main()


#!/usr/bin/env python

import argparse
import csv
import datetime
import functools
import os
import sys
from collections import defaultdict
from multiprocessing import Pool, Pipe
from multiprocessing import cpu_count
from multiprocessing.context import TimeoutError

# from tqdm import tqdm
import numpy as np
import pysam
from interlap import InterLap
from rich import print
from rich.progress import (
    BarColumn,
    TimeRemainingColumn,
    Progress,
)

progress = Progress(
    "[progress.description]{task.description}",
    BarColumn(),
    "[progress.percentage]{task.percentage:>3.0f}%",
    "ETA:",
    TimeRemainingColumn()
)

progressupdates_recv, progressupdates_send = Pipe()

defaultchroms = list(map(str, range(1, 720))) + ['X', 'Y', '2L', '2R', '3L', '3R', 'Z', 'W', 'I', 'II', 'III', 'IV',
                                                 'V', 'VI',
                                                 'VII', 'VIII', 'IX', 'X', 'XI', 'XII', 'XIII', 'XIV', 'XV', 'XVI']

def exon2intron(listOfExons, min_intron_length):
    """
    exons must be zero-based, start excluded
    :param listOfExons:
    :param min_intron_length:
    :return: are zero-based, start excluded
    """

    minus_strand = listOfExons[0][3] == "-"
    listOfExons.sort(key=lambda exon: exon[1])
    numberExons = len(listOfExons)
    intron = []
    if numberExons > 1:
        strandrange = range(1, numberExons) if not minus_strand else reversed(range(1, numberExons))
        id = 0
        for i in strandrange:
            exon0 = listOfExons[i - 1]
            exon1 = listOfExons[i]
            id += 1
            thisintron = (exon0[0], exon0[2], exon1[1], exon1[3], id)
            if (thisintron[2] - thisintron[1] - 1 >= min_intron_length):
                intron.append(thisintron)
    return intron

def get_sjCoordinates(GTFfile, min_intron_length=30, keep_masked_introns=False, filterSJs=False, intron_file=None,
                      exon_file=None, chroms=defaultchroms):
    # gets transcript per gene
    iso2gene = {}
    # lists exons per transcript (tcx)
    iso2exons = defaultdict(list)
    exon_by_chr = defaultdict(dict)

    for line in GTFfile:
        splitGTFline = line.split('\t')
        chrom = splitGTFline[0]
        if len(chrom) > 3 and chrom[:3] == "chr":
            chrom = chrom[3:]

        if chrom != '#' and chrom in chroms:
            if splitGTFline[2] == 'exon':
                gene = line.split('gene_id')[1].split('"')[1]
                tcx = line.split('transcript_id')[1].split('"')[1]
                iso2gene[tcx] = gene
                start1 = int(splitGTFline[3])
                end1 = int(splitGTFline[4])
                strand = splitGTFline[6]
                newExon = (chrom, start1 - 1, end1, strand)
                iso2exons[tcx].append(newExon)
                if not keep_masked_introns or filterSJs:
                    if chrom not in exon_by_chr:
                        exon_by_chr[chrom] = defaultdict(InterLap)
                    exon_by_chr[chrom][strand].add((start1, end1, {'g': gene}))
                if exon_file is not None and not exon_file.closed:
                    exon_file.write("\t".join([chrom, str(start1 - 1), splitGTFline[4], gene + "_" + tcx, "0",
                                               strand]) + "\n")  # bed structure

    GTFfile.close()

    # calculates intron location based on exons for each tcx
    introns = defaultdict(list)
    for tcx, listOfExons in iso2exons.items():
        # gets list of introns (no UTR)
        listOfIntrons = exon2intron(listOfExons, min_intron_length)
        for intron in listOfIntrons:
            if keep_masked_introns:
                introns[tcx].append(intron)
            else:
                # Only if this intron interval is not in the list of exons intervals for this chr-strand pair: add it
                if not (intron[1] + 1, intron[2]) in exon_by_chr[intron[0]][intron[3]]:
                    introns[tcx].append(intron)

    sjIntervals = defaultdict(dict)
    intronIDtoTcxGeneNr = defaultdict()
    intronID = 0
    for tcx, listOfIntrons in introns.items():
        for intron in listOfIntrons:
            gene = iso2gene[tcx]

            if intron_file is not None and not intron_file.closed:
                joined = "\t".join([intron[0], str(intron[1]), str(intron[2]), gene + "_" + tcx, "0", intron[3]]) + "\n"
                intron_file.write(joined)

            if intron[3] == '+':
                if filterSJs:
                    overlap5 = False
                    for overlappingExon in exon_by_chr[intron[0]][intron[3]].find((intron[1], intron[1] + 1)):
                        if overlappingExon[2]['g'] != gene:
                            overlap5 = True
                            break

                    overlap3 = False
                    for overlappingExon in exon_by_chr[intron[0]][intron[3]].find((intron[2], intron[2] + 1)):
                        if overlappingExon[2]['g'] != gene:
                            overlap3 = True
                            break

                    if not overlap3 and not overlap5:
                        if intron[0] not in sjIntervals:
                            sjIntervals[intron[0]] = defaultdict(InterLap)
                        sjIntervals[intron[0]][intron[3]].add(
                            (intron[1], intron[1] + 1, {'sj3': False, 's': 0, 'us': 0, 'id': intronID}))
                        sjIntervals[intron[0]][intron[3]].add(
                            (intron[2], intron[2] + 1, {'sj3': True, 's': 0, 'us': 0, 'id': intronID}))

                        intronIDtoTcxGeneNr[intronID] = gene + "_" + tcx + "_" + str(intron[4])
                        intronID += 1
                else:
                    if intron[0] not in sjIntervals:
                        sjIntervals[intron[0]] = defaultdict(InterLap)
                    sjIntervals[intron[0]][intron[3]].add(
                        (intron[1], intron[1] + 1, {'sj3': False, 's': 0, 'us': 0, 'id': intronID}))
                    sjIntervals[intron[0]][intron[3]].add(
                        (intron[2], intron[2] + 1, {'sj3': True, 's': 0, 'us': 0, 'id': intronID}))
                    intronIDtoTcxGeneNr[intronID] = gene + "_" + tcx + "_" + str(intron[4])
                    intronID += 1
            else:
                if filterSJs:
                    overlap5 = False
                    for overlappingExon in exon_by_chr[intron[0]][intron[3]].find((intron[2], intron[2] + 1)):
                        if overlappingExon[2]['g'] != gene:
                            overlap5 = True
                            break

                    overlap3 = False
                    for overlappingExon in exon_by_chr[intron[0]][intron[3]].find((intron[1], intron[1] + 1)):
                        if overlappingExon[2]['g'] != gene:
                            overlap3 = True
                            break

                    if not overlap3 and not overlap5:
                        if intron[0] not in sjIntervals:
                            sjIntervals[intron[0]] = defaultdict(InterLap)
                        sjIntervals[intron[0]][intron[3]].add(
                            (intron[2], intron[2] + 1, {'sj3': False, 's': 0, 'us': 0, 'id': intronID}))
                        sjIntervals[intron[0]][intron[3]].add(
                            (intron[1], intron[1] + 1, {'sj3': True, 's': 0, 'us': 0, 'id': intronID}))

                        intronIDtoTcxGeneNr[intronID] = gene + "_" + tcx + "_" + str(intron[4])
                        intronID += 1
                else:
                    if intron[0] not in sjIntervals:
                        sjIntervals[intron[0]] = defaultdict(InterLap)
                    sjIntervals[intron[0]][intron[3]].add(
                        (intron[2], intron[2] + 1, {'sj3': False, 's': 0, 'us': 0, 'id': intronID}))
                    sjIntervals[intron[0]][intron[3]].add(
                        (intron[1], intron[1] + 1, {'sj3': True, 's': 0, 'us': 0, 'id': intronID}))
                    intronIDtoTcxGeneNr[intronID] = gene + "_" + tcx + "_" + str(intron[4])
                    intronID += 1

    return sjIntervals, intronID, intronIDtoTcxGeneNr

def processBamEntry(chromSJ, bamfile, verbose, read_qual, chr_prefix):
    with open(bamfile, 'r') as bamf:

        chrom, sjintervals = chromSJ
        contig = chrom
        if chr_prefix:
            contig = "chr" + chrom
        if verbose:
            print("Processing chromosome " + chrom + " ...")

        bam = pysam.AlignmentFile(bamf, 'rb')
        for read in bam.fetch(contig=contig):
            if read.mapping_quality > read_qual:
                strand = "+"
                if read.is_reverse:
                    strand = "-"

                # zero-based left-most coordinate. In the Interval tree they are stored one-based.
                readStart = read.reference_start + 1
                # readStart = read.reference_start
                # pysam reports the end non-inclusive
                readEnd = read.reference_end - 1

                countIdx = 6
                key = 'us'  # nonsplit
                for cigaroperation, length in read.cigartuples:
                    if cigaroperation == 3:  # if any M present
                        countIdx = 7
                        key = 's'  # split
                        break

                if countIdx == 6:  # nonsplit reads
                    for mapped_sj_interval in sjintervals[strand].find((readStart, readEnd)):
                        if (readEnd >= mapped_sj_interval[1] and
                                readStart <= mapped_sj_interval[0]):
                            mapped_sj_interval[2][key] += 1
                else:  # split reads

                    for blockstart, blockend in read.get_blocks():
                        for mapped_sj_interval in sjintervals[strand].find((blockstart, blockend)):
                            if ((blockstart + 1) == mapped_sj_interval[1]) or (blockend == mapped_sj_interval[0]):
                                mapped_sj_interval[2][key] += 1

        if verbose:
            print("Processing chromosome " + chrom + " done.")

        bam.close()
        return chrom, sjintervals

## For use if we support multiple Bam files.
def resetSJDict(sjintervals):
    for chrom, strands in sjintervals.items():
        for strand, intervals in strands.items():
            for interval in intervals:
                interval[2]['us'] = 0
                interval[2]['s'] = 0

######## -------------------------  IER ----------- #############

def calcScore(introncov, leftexoncov, rightexoncov):
    exonsumm = (leftexoncov + rightexoncov)
    if exonsumm == 0:
        return 1.
    else:
        return 1. - min(1, (introncov / (0.5 * exonsumm)))

def sameChrom(tcxExons, chromNr):
    tcx, listExons = tcxExons
    return chromNr == listExons[0][0]

def splitIntoChroms(tcxExons, chroms):
    res = [] * len(chroms)
    i = 0
    for chrom in chroms:
        for tcxexon in tcxExons:
            if sameChrom(tcxexon, chrom):
                res[i].append(tcxexon)
        i += 1
    return res

def sjOverlaps(intron, exon_by_chr, gene):
    for overlappingExon in exon_by_chr[(intron[0], intron[3])].find((intron[1], intron[1] + 1)):
        if overlappingExon[2]['g'] != gene:
            return True

    for overlappingExon in exon_by_chr[(intron[0], intron[3])].find((intron[2], intron[2] + 1)):
        if overlappingExon[2]['g'] != gene:
            return True

def getCoverage(bam, chrom, start, end, minus_strand, min_quality):
    getCoverage.cache = getattr(getCoverage, 'cache', tuple((defaultdict(), defaultdict())))
    if (start, end) in getCoverage.cache[minus_strand]:
        return getCoverage.cache[minus_strand][(start, end)]

    if minus_strand:
        filterRead = lambda r: r.is_reverse and r.mapq > min_quality
    else:
        filterRead = lambda r: not r.is_reverse and r.mapq > min_quality

    ## sum up coverages per nucleotide
    cov_array = functools.reduce(np.add,
                                 bam.count_coverage(contig='chr' + str(chrom), start=start,
                                                    stop=end, quality_threshold=0,
                                                    read_callback=filterRead))
    ## return median coverage for this genomic feature/interval
    result = np.median(cov_array)
    getCoverage.cache[minus_strand][(start, end)] = result
    return result

def getSJCoverage(bam, chrom, start, end, minus_strand, min_quality, debug):
    """
    Gets the coverage for split and non-split reads in the bam file

    @type  start: int
    @param start: 0-based inclusive start position of SJ
    @type  end: int
    @param end: 0-based exclusive start position of SJ
    @rtype:   tuple(int,int)
    @return:  number split, number non-split reads
    """
    getSJCoverage.cache = getattr(getSJCoverage, 'cache', tuple((defaultdict(), defaultdict())))
    if (start, end) in getSJCoverage.cache[minus_strand]:
        return getSJCoverage.cache[minus_strand][(start, end)]

    def filterReadSplit(read):
        # read quality also covers "uniquely mapped" etc. otherwise we could separately check flags
        if read.mapping_quality <= min_quality:
            return False
        if read.is_reverse == minus_strand:
            for cigaroperation, length in read.cigartuples:
                if cigaroperation == 3:
                    return read.get_overlap(start, end) == 1
            return False
        else:
            return False

    def filterReadUnsplit(read):
        if read.mapq <= min_quality:
            return False
        if read.is_reverse == minus_strand:
            for cigaroperation, length in read.cigartuples:
                if cigaroperation == 3:
                    return False
            # Full overlap
            return read.aend > end and read.pos <= start
        else:
            return False

    array = bam.count_coverage(contig='chr' + str(chrom), start=start,
                               stop=end, quality_threshold=0,
                               read_callback=filterReadSplit)

    cov_split_array = functools.reduce(np.add, array)

    cov_unsplit_array = functools.reduce(np.add,
                                         bam.count_coverage(contig='chr' + str(chrom), start=start,
                                                            stop=end, quality_threshold=0,
                                                            read_callback=filterReadUnsplit))

    result = np.max(cov_split_array), np.max(cov_unsplit_array)
    getSJCoverage.cache[minus_strand][(start, end)] = result
    return result

def worker_main(itemsexonschrom, bamfile, chr_prefix, verbose, min_intron_length, min_coverage, min_quality):
    items, exons, chrom, chromnr, task_id = itemsexonschrom

    if chr_prefix:
        contig = "chr" + chrom
    else:
        contig = chrom

    bam = pysam.AlignmentFile(bamfile, 'rb')
    n = len(items.items())
    # print(n, "transcripts on chromosome ", contig)
    resultrows = []

    progressupdates_send.send((task_id, True, n))

    updateinterval = 100
    c = 0
    # descr = "Chromosome " + str(chrom)
    # with tqdm(total=n, desc=descr, position=chromnr, leave=False) as pbar:
    for tcxExons in items.items():
        c += 1
        if c % updateinterval == 0:
            progressupdates_send.send((task_id, False, updateinterval))
        # pbar.update(1)
        tcx, listOfExons = tcxExons

        numberExons = len(listOfExons)

        listOfExons.sort(key=lambda exon: exon[1])
        minus_strand = listOfExons[0][3] == "-"

        strandrange = range(1, numberExons) if not minus_strand else reversed(range(1, numberExons))
        intronid = 0

        exoncov3p = None
        chrom = listOfExons[0][0]
        strand = listOfExons[0][3]

        for i in strandrange:

            exoncov5p = exoncov3p
            exoncov3p = None

            if minus_strand:
                exon5p = listOfExons[i]
                exon3p = listOfExons[i - 1]
            else:
                exon5p = listOfExons[i - 1]
                exon3p = listOfExons[i]

            intronid += 1
            debug = False

            thisintron = (listOfExons[i - 1][0], listOfExons[i - 1][2], listOfExons[i][1], listOfExons[i][3], intronid)

            # filter by intron length
            if (thisintron[2] - thisintron[1] - 1 >= min_intron_length):
                # filter by overlap with other exons (i.e. level 3)
                if (thisintron[1] + 1, thisintron[2]) not in exons[strand]:
                    if exoncov5p is None:
                        exoncov5p = getCoverage(bam, chrom, exon5p[1], exon5p[2], minus_strand, min_quality)

                    exoncov3p = getCoverage(bam, chrom, exon3p[1], exon3p[2], minus_strand, min_quality)

                    introncov = getCoverage(bam, chrom, thisintron[1], thisintron[2], minus_strand, min_quality)
                    sjcov5p_s, sjcov5p_us = getSJCoverage(bam, chrom,
                                                          exon5p[2 - int(minus_strand)] - 1,
                                                          exon5p[2 - int(minus_strand)] + 1,
                                                          minus_strand, min_quality, debug)
                    sjcov3p_s, sjcov3p_us = getSJCoverage(bam, chrom,
                                                          exon3p[1 + int(minus_strand)] - 1,
                                                          exon3p[1 + int(minus_strand)] + 1,
                                                          minus_strand, min_quality, debug)

                    if (sjcov5p_s + sjcov5p_us >= min_coverage and
                            sjcov3p_s + sjcov3p_us >= min_coverage):
                        resultrows.append([thisintron[0], thisintron[1], thisintron[2],
                                           thisintron[3], thisintron[4],
                                           None, tcx, exoncov5p, sjcov5p_s, sjcov5p_us,
                                           introncov, sjcov3p_s,
                                           sjcov3p_us, exoncov3p,
                                           calcScore(introncov, exoncov5p, exoncov3p)])

    progressupdates_send.send(
        (task_id, False, updateinterval))  # send another batch to definitely set the progress to 100%
    return resultrows

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-g', '--gtffile', metavar='', type=argparse.FileType('r'), nargs=1, required=True,
                        help="GTFfile provided by GENCODE or Ensembl.")
    parser.add_argument('-b', '--bamfile', metavar='', type=argparse.FileType('r'), nargs=1, required=True,
                        help="Sorted BAM file.")
    parser.add_argument('-o', '--outfile', metavar='', type=str,
                        help="Output file name.")
    parser.add_argument('-q', '--quiet', action='store_true',
                        help="Enable quiet mode (only shows warnings and errors).")
    parser.add_argument('-i', '--IERatio', action='store_true',
                        help="Running mode that additionally outputs the Inverse Intron Expression Ratio (IER). Takes longer.")
    parser.add_argument('-f', '--filterLevel', type=int, choices=range(1, 4), metavar='', default=3,
                        help="Levels:"
                             "1) Keeps all introns in the genome. "
                             "2) Keeps only introns whose SJs do not overlap with exons in different genes. "
                             "3) Keeps only introns that do not overlap with exons in other parts of the genome (Default).")
    parser.add_argument('-l', '--MinIntronLength', type=int, metavar='', default=30,
                        help="Minimum intron length. (Default = 30)")
    parser.add_argument('-r', '--MinReadQuality', type=int, metavar='', default=10,
                        help="Minimum read quality. (Default > 10)")
    parser.add_argument('-c', '--MinCoverage', type=int, metavar='', default=10,
                        help="Minimum coverage of each splice junctions. (Default = 10)")
    parser.add_argument('-p', '--NrProcesses', type=int, metavar='', default=4,
                        help="Number of processes spawned to process chromosomes in parallel. 0 spawns as many as processors are available.")
    parser.add_argument('-x', '--ChromsList', type=str, metavar='', default='',
                        help="List of Chromosome names separated by comma (without spaces). Without this parameter a default set of common chromosomes is used.")

    args = parser.parse_args()

    global defaultchroms
    if not args.ChromsList == '':
        defaultchroms = args.ChromsList.split(',')

    bamfiles = args.bamfile
    for bamfile in bamfiles:
        bamfile_name = bamfile.name

        if not os.path.isfile(bamfile_name + ".bai") or os.stat(bamfile_name).st_mtime >= os.stat(
                bamfile_name + ".bai").st_mtime:
            now = datetime.datetime.now()
            print(now.strftime("%D   %H:%M") + '   Generating index file ' + bamfile_name + ".bai" + "...")
            bai = str(bamfile_name + ".bai")
            pysam.index(bamfile_name, bai)
        else:
            print("Index available")
            continue

    if args.IERatio:
        if not args.filterLevel == 3:
            print("Warning: using IER is restricted to --filterLevel 3. Ignoring user input for -f/--filterLevel.")
        main_coverage(parser)
    else:
        main_nocoverage(parser)

def main_nocoverage(parser):
    args = parser.parse_args()
    verbose = not args.quiet
    gtffile = args.gtffile[0]
    bamfiles = args.bamfile
    if gtffile == None or bamfiles == None:
        print("ERROR: Check input file arguments!", file=sys.stderr)
        parser.print_help()
        exit(1)

    if verbose:
        now = datetime.datetime.now()
        print(now.strftime("%D   %H:%M") + '   Parsing splice junctions...')

    keepMaskedIntrons = args.filterLevel <= 2
    filterSJs = args.filterLevel == 2
    sjintervals, nrIntrons, intronIDtoTcxGeneNr = get_sjCoordinates(gtffile, min_intron_length=args.MinIntronLength,
                                                                    keep_masked_introns=keepMaskedIntrons,
                                                                    filterSJs=filterSJs,
                                                                    intron_file=None,
                                                                    exon_file=None,
                                                                    chroms=defaultchroms)

    nrProcesses = cpu_count()
    if args.NrProcesses > 0:
        nrProcesses = args.NrProcesses

    for bamfile in bamfiles:
        # Open bam quickly to check chromosomes to be extracted
        name = bamfile.name
        bam = pysam.AlignmentFile(bamfile, 'rb')
        bamRefs = set()
        chr_prefix = False

        for r in bam.references:
            if r[:3] == "chr":
                chr_prefix = True
                bamRefs.add(r[3:])
            else:
                bamRefs.add(r)
        foundChroms = set(defaultchroms).intersection(bamRefs)
        bam.close()

        if nrProcesses > len(foundChroms):
            nrProcesses = len(foundChroms)
            print("Warning: Number of requested processes is higher than the number of chromosomes to "
                  "parallelize over. Reducing to " + str(nrProcesses))

        if verbose:
            now = datetime.datetime.now()
            print(now.strftime("%D   %H:%M") + '   Calculating coverage for ' + name + '...')

        read_qual = args.MinReadQuality

        processBamEntryWSettings = functools.partial(processBamEntry, bamfile=name, verbose=verbose,
                                                     read_qual=read_qual, chr_prefix=chr_prefix)
        p = Pool(processes=nrProcesses)
        results = p.map(processBamEntryWSettings, [(x, sjintervals[x]) for x in foundChroms if x in sjintervals])
        p.close()
        p.join()

    # Initialize output table
    outfilename = args.outfile
    if outfilename is None or len(outfilename) == 0:
        outfilename = "splicing-efficiency-score.tsv"

    # rows: nrIntrons
    # cols: 13, see header line below
    lines = [[None for x in range(14)] for y in range(nrIntrons)]

    # process file...
    # Fill the entries at the row corresponding to the ID that was saved for the SJ
    for chrom, strands in results:
        for strand, intervals in strands.items():
            for interval in intervals:
                currentIntronID = interval[2]['id']
                entry = lines[currentIntronID]
                if interval[2]['sj3']:
                    entry[0] = chrom
                    entry[1] = strand
                    idColumn = intronIDtoTcxGeneNr[currentIntronID].split("_")
                    entry[2] = idColumn[0]
                    entry[3] = idColumn[1]
                    entry[4] = idColumn[2]
                    offset = 9
                else:
                    offset = 5
                entry[offset] = interval[0]
                entry[offset + 1] = interval[1]
                entry[offset + 2] = interval[2]['us']
                entry[offset + 3] = interval[2]['s']

    if verbose:
        now = datetime.datetime.now()
        print(now.strftime("%D   %H:%M") + '   Calculation splicing efficiencies...')

    if verbose:
        now = datetime.datetime.now()
        print(now.strftime("%D   %H:%M") + '   Writing output...')

    skipped = 0
    min_coverage_param = args.MinCoverage
    min_coverage = max(1, min_coverage_param)
    with open(outfilename, 'w') as f:
        # header
        f.write("\t".join(
            ["chr", "strand", "gene_ID", "transcript_ID", "intron_ID",
             "sj5start", "sj5end", "sj5_cov_nonsplit", "sj5_cov_split",
             "sj3start", "sj3end", "sj3_cov_nonsplit", "sj3_cov_split", "score"]) + "\n")
        for line in lines:
            # skip filtered entries
            if line[7] is None or line[8] is None or line[11] is None or line[12] is None:
                skipped = skipped + 1
                continue

            sj5_cov = float(line[7] + line[8])
            sj3_cov = float(line[11] + line[12])

            if sj5_cov >= min_coverage and sj3_cov >= min_coverage:
                score = (float(line[8] + line[12])) / (sj5_cov + sj3_cov)
                line[13] = str(score)
                f.write("\t".join(str(x) for x in line) + "\n")
            # else skip this intron completely
    f.close()

    if verbose:
        if skipped > 0:
            "Warning: Skipped " + str(skipped) + " introns (probably due to missing chromosome/contig in bam file)."
        print("Done!")

def progress_checking(recv):
    # for task_id, rec in recv:
    if recv.poll():
        task_id, init, inc = recv.recv()
        if init:
            progress.update(task_id, total=inc)
            progress.start_task(task_id)
        else:
            progress.advance(task_id, inc)

def main_coverage(parser):
    args = parser.parse_args()

    verbose = not args.quiet
    GTFfile = args.gtffile[0]
    bamfile = args.bamfile[0]
    if GTFfile is None or bamfile is None:
        print("ERROR: Check input file arguments!", file=sys.stderr)
        parser.print_help()
        exit(1)

    if verbose:
        now = datetime.datetime.now()
        print(now.strftime("%D   %H:%M") + '   Parsing splice junctions...')

    min_quality = args.MinReadQuality

    nrProcesses = cpu_count()
    if args.NrProcesses > 0:
        nrProcesses = args.NrProcesses

    min_intron_length = args.MinIntronLength
    min_coverage_param = args.MinCoverage
    min_coverage = max(1, min_coverage_param)

    chr_prefix = False

    bam = pysam.AlignmentFile(bamfile.name, 'rb')
    bamRefs = set()

    for r in bam.references:
        if r[:3] == "chr":
            chr_prefix = True
            bamRefs.add(r[3:])
        else:
            bamRefs.add(r)
    foundChroms = set(defaultchroms).intersection(bamRefs)
    bam.close()

    # gets transcript per gene
    iso2gene = {}
    # lists exons per transcript (tcx)
    chrom2iso2exons = defaultdict(defaultdict)
    exon_by_chr = defaultdict(dict)
    # tcxpos = defaultdict()

    # get exons from GTF
    for line in GTFfile:
        splitGTFline = line.split('\t')
        chrom = splitGTFline[0]
        if len(chrom) > 3 and chrom[:3] == "chr":
            chrom = chrom[3:]

        if chrom != '#' and chrom in defaultchroms:
            if splitGTFline[2] == 'exon':
                gene = line.split('gene_id')[1].split('"')[1]
                tcx = line.split('transcript_id')[1].split('"')[1]
                iso2gene[tcx] = gene
                start1 = int(splitGTFline[3])
                end1 = int(splitGTFline[4])
                strand = splitGTFline[6]
                newExon = (chrom, start1 - 1, end1, strand)
                if chrom not in chrom2iso2exons:
                    chrom2iso2exons[chrom] = defaultdict(list)
                chrom2iso2exons[chrom][tcx].append(newExon)
                if chrom not in exon_by_chr:
                    exon_by_chr[chrom] = defaultdict(InterLap)
                exon_by_chr[chrom][strand].add((start1, end1, {'g': gene}))

    GTFfile.close()

    # Initialize output table

    outfilename = args.outfile
    if len(outfilename) == 0:
        outfilename = "EIratio.tsv"

    # with Pool(processes=nrProcesses, initargs=(RLock(),), initializer=tqdm.set_lock) as p:
    with Pool(processes=nrProcesses) as p:
        with open(outfilename, 'w+') as f:
            writer = csv.writer(f, lineterminator='\n', delimiter="\t")
            writer.writerow(["chr", "IStart", "IEnd", "strand", "intron_ID", "gene_ID", "transcript_ID",
                             "exon5_cov", "sj5_cov_split",
                             "sj5_cov_nonsplit", "intron_cov", "sj3_cov_split", "sj3_cov_nonsplit",
                             "exon3_cov", "IER"])
            now = datetime.datetime.now()
            print(now.strftime("%D   %H:%M") + '   Calculating coverage...')
            chromtotask = {}
            for chrom in foundChroms:
                chromtotask[chrom] = progress.add_task("Chromosome " + str(chrom), start=False)

            progress.start()
            processtcxwsettings = functools.partial(worker_main, bamfile=bamfile.name, chr_prefix=chr_prefix,
                                                    verbose=verbose, min_intron_length=min_intron_length,
                                                    min_coverage=min_coverage, min_quality=min_quality)

            it = p.imap_unordered(processtcxwsettings,
                                  [(chrom2iso2exons[chrom], exon_by_chr[chrom], chrom, i, chromtotask[chrom]) for
                                   i, chrom in enumerate(foundChroms)])
            while True:
                try:
                    result = it.next(timeout=0.1)
                except TimeoutError:
                    progress_checking(progressupdates_recv)
                except StopIteration:
                    break
                except IndexError:
                    break
                else:
                    if result is not None:
                        for row in result:
                            row[5] = iso2gene[row[6]]
                            writer.writerow(row)
            progress.stop()
            f.close()

    now = datetime.datetime.now()
    print(now.strftime("%D   %H:%M") + '   Done!')

if __name__ == '__main__':
    main()


from typing import List, Tuple, Any, Dict, Optional
from ..utils import *
from ..event_series import *

POS_ROUND_DIGITS = 3
SOFT_COLLIDE_SHIFT = 0.2

BAG_RADIUS = 0.5
BAG_MASS = 20
BAG_DENSITY = 40

BELT_WIDTH = 1.5
BELT_UNIT_MASS = 15

SPEED_STEP = 0.1
SPEED_ROUND_DIGITS = 3

MODEL_BELT_LEN = 313.25

_G = 9.8
_FRICTON_COEFF = 0.024
_THETA_1 = 1/(6.48*BELT_WIDTH*BELT_WIDTH*BAG_DENSITY)
_THETA_2_L = _G*_FRICTON_COEFF*BELT_UNIT_MASS
_k_3 = 4000
_THETA_3 = 0.0031
_THETA_4_L = _G*_FRICTON_COEFF
_k_2 = 30

_ETA = 0.8

# THETAS_ORIG = [2.3733e-4, 8566.3, 0.0031, 51.6804]

def _P_Zhang(length, V, T):
    return _THETA_1*V*T*T + (_THETA_2_L*length + _k_3)*V + \
        _THETA_3*T*T/V + (_THETA_4_L*length + _k_2)*T + V*V*T/3.6

def consumption_Zhang(length, speed, n_bags):
    if speed == 0:
        return 0
    Q_g = n_bags * BAG_MASS / length
    T = Q_g * speed * 3.6
    return _P_Zhang(length, speed, T) / _ETA

def consumption_linear(length, speed, n_bags):
    return _G * _FRICTION_COEFF * BAG_MASS * n_bags * speed / _ETA

##
# ^ weird stuff up here ...
#

class EnergySpender(ChangingValue):
    """
    Class which records energy consumption
    """
    def __init__(self, data_series: EventSeries, length, consumption='zhang'):
        super().__init__(data_series)
        if type(consumption) == 'float':
            self._consumption = lambda v, n: consumption
        elif consumption == 'zhang':
            self._consumption = lambda v, n: consumption_Zhang(length, v, n)
        elif consumption == 'linear':
            self._consumption = lambda v, n: consumption_linear(length, v, n)
        else:
            raise Exception('consumption: ' + consumption)

    def update(self, time, speed, n_bags):
        super().update(time, self._consumption(speed, n_bags))

class CollisionException(Exception):
    """
    Thrown when objects on conveyor collide
    """
    def __init__(self, args):
        super().__init__('[{}] Objects {} and {} collided in position {}'
                        .format(args[3], args[0], args[1], args[2]))
        self.obj1 = args[0]
        self.obj2 = args[1]
        self.pos = args[2]
        self.handler = args[3]

class AutomataException(Exception):
    pass

def search_pos(ls: List[Tuple[Any, float]], pos: float,
               return_index: bool = False,
               preference: str = 'nearest') -> Tuple[Any, float]:
    assert len(ls) > 0, "what are you gonna find pal?!"
    return binary_search(ls, differs_from(pos, using=lambda p: p[1]),
                         return_index=return_index, preference=preference)

def shift(objs, d):
    return [(o, round(p + d, POS_ROUND_DIGITS)) for (o, p) in objs]

# Automata for simplifying the modelling of objects movement
_model_automata = {
    'pristine': {'resume': 'moving', 'change': 'dirty'},
    'moving': {'pause': 'pristine'},
    'dirty': {'change': 'dirty', 'start_resolving': 'resolving'},
    'resolving': {'change': 'resolving', 'end_resolving': 'resolved'},
    'resolved': {'resume': 'moving'}
}

class ConveyorModel:
    """
    Datatype which allows for modeling the conveyor with
    objects moving on top of it.

    A conveyor is modeled as a line with some checkpoints
    placed along it. Only one checkpoint can occupy a given position.

    Objects can be placed on a conveyor. All the objects on a conveyor
    move with the same speed - the current speed of a conveyor. Current speed
    of a conveyor can be changed.

    We want to have the following operations:
    - put an object to a conveyor (checking for collisions)
    - remove an object from a conveyor
    - change the conveyor speed
    - update the positions of objects as if time T has passed with a current
      conveyor speed (T might be negative, moving objects backwards)
    - calculate time T until earliest event of object reaching a checkpoint
      (or the end of the conveyor), and also return those checkpoint and object
    """
    def __init__(self, env: DynamicEnv, length: float, max_speed: float,
                 checkpoints: List[Tuple[Any, float]], energy_series: EventSeries = None,
                 model_id: AgentId = ('world', 0)):
        assert length > 0, "Conveyor length <= 0!"
        assert max_speed > 0, "Conveyor max speed <= 0!"

        checkpoints = sorted(checkpoints, key=lambda p: p[1])
        if len(checkpoints) > 0:
            assert checkpoints[0][1] >= 0, "Checkpoints with position < 0!"
            assert checkpoints[-1][1] < length, "Checkpoint with position >= conveyor length!"
            for i in range(len(checkpoints) - 1):
                assert checkpoints[i][1] < checkpoints[i+1][1], \
                    "Checkpoints with equal positions!"

        # constants
        self.env = env
        self.model_id = model_id
        self.checkpoints = checkpoints
        self.checkpoint_positions = {cp: pos for (cp, pos) in checkpoints}
        self.length = length
        self.max_speed = max_speed

        # variables
        self.speed = 0
        self.objects = {}
        self.object_positions = []

        self._spender = None
        if energy_series is not None:
            self._spender = EnergySpender(energy_series, self.length)

        self._state = 'pristine'
        self._resume_time = 0
        self._resolved_events = set()
        self._speed_changes = 0
        self._speed_sum = 0

    def _stateTransfer(self, action):
        try:
            self._state = _model_automata[self._state][action]
            if action == 'change' and self._spender is not None:
                self._spender.update(self.env.time(), self.speed, len(self.objects))
        except KeyError:
            print('RAISED')
            raise AutomataException(
                '{}: Invalid action `{}` in state `{}`;\n  speed - {}m/s\n  cps - {};\n  objs - {}'
                .format(self.model_id, action, self._state, self.speed,
                        self.checkpoints, self.object_positions))

    def totalEnergySpent(self):
        return self._spender.total(self.env.time())

    def checkpointPos(self, cp: Any) -> Optional[float]:
        if cp[0] == 'conv_end':
            return self.length
        return self.checkpoint_positions.get(cp, None)

    def nextCheckpoint(self, pos: float) -> Tuple[Any, float]:
        return search_pos(self.checkpoints, pos, preference='next')

    def nearestObject(self, pos: float, after=None, speed=None, not_exact=False,
                      preference='nearest') -> Optional[Tuple[Any, float]]:
        if len(self.object_positions) == 0:
            return None

        if after is not None:
            if speed is None:
                speed = self.speed
            objs = shift(self.object_positions, after * speed)
        else:
            objs = self.object_positions

        res = search_pos(objs, pos, preference=preference, return_index=True)
        if res is not None:
            (oid, o_pos), idx = res
            if not_exact and o_pos == pos:
                if preference == 'prev':
                    idx -= 1
                elif preference == 'next':
                    idx += 1
                else:
                    raise Exception('please dont use nearest with not exact')
                if idx < 0 or idx > len(objs):
                    return None
                oid, o_pos = objs[idx]
            return self.objects[oid], o_pos
        return None

    def working(self) -> bool:
        return self.speed > 0

    def setSpeed(self, speed: float):
        assert speed >= 0 and speed <= self.max_speed, \
            "Speed is not in [0, max_speed]!"
        if self.speed != speed:
            self._stateTransfer('change')
            self.speed = speed
            self._speed_changes += 1
            self._speed_sum += speed

    def putObject(self, obj_id: int, obj: Any, pos: float, soft_collide=True, return_nearest=False):
        assert obj_id not in self.objects, "Clashing object ID!"
        pos = round(pos, POS_ROUND_DIGITS)

        nearest = None
        if len(self.objects) > 0:
            (n_obj_id, n_pos), n_idx = search_pos(self.object_positions, pos, return_index=True)
            if n_pos == pos:
                if soft_collide:
                    print('{}: TRUE COLLISION: #{} and #{} on {}'.format(self.model_id, obj_id, n_obj_id, pos))
                    i = n_idx
                    p_pos = pos
                    while i < len(self.object_positions) and self.object_positions[i][1] >= p_pos:
                        p_pos = round(p_pos + SOFT_COLLIDE_SHIFT, POS_ROUND_DIGITS)
                        self.object_positions[i] = (self.object_positions[i][0], p_pos)
                        i += 1
                else:
                    raise CollisionException((obj, self.objects[n_obj_id], pos, self.model_id))
            elif n_pos < pos:
                n_idx += 1
            nearest = (n_obj_id, n_pos)
        else:
            n_idx = 0

        self.objects[obj_id] = obj
        self.object_positions.insert(n_idx, (obj_id, pos))
        self._stateTransfer('change')

        if return_nearest:
            return nearest

    def objPos(self, obj_id: int):
        for (oid, pos) in self.object_positions:
            if oid == obj_id:
                return pos
        return None

    def removeObject(self, obj_id: int):
        pos_idx = None
        for (i, (oid, pos)) in enumerate(self.object_positions):
            if oid == obj_id:
                pos_idx = i
                break

        self.object_positions.pop(pos_idx)
        obj = self.objects.pop(obj_id)
        self._stateTransfer('change')
        return obj

    def shift(self, d):
        self.object_positions = shift(self.object_positions, d)

    def skipTime(self, time: float, clean_ends=True):
        if time == 0:
            return 0

        self._stateTransfer('change')
        d = time * self.speed
        if len(self.objects) == 0:
            return d

        last_pos_orig = self.object_positions[-1][1]
        self.shift(d)

        if clean_ends:
            while len(self.object_positions) > 0 and self.object_positions[0][1] < 0:
                obj_id, _ = self.object_positions.pop(0)
                self.objects.pop(obj_id)
            while len(self.object_positions) > 0 and self.object_positions[-1][1] > self.length:
                obj_id, pos = self.object_positions.pop()
                # print('HAHA NE ZHDAL? A VOT, NAHUI IDET: {}, {} -> {}; {}s, {}m/s'.format(obj_id, last_pos_orig, pos, time, self.speed))
                self.objects.pop(obj_id)

        return d

    def nextEvents(self, obj=None, cps=None, skip_immediate=True,
                   skip_resolved=True) -> List[Tuple[Any, Any, float]]:
        if self.speed == 0:
            return []

        obj = def_list(obj, self.objects.keys())
        obj_positions = [(oid, pos) for (oid, pos) in self.object_positions
                         if oid in obj]

        cps = def_list(cps, self.checkpoint_positions.keys())
        c_points = [(cp, pos) for (cp, pos) in self.checkpoints if cp in cps]
        c_points.append((('conv_end', self.model_id[1]), self.length))

        def _skip_cond(obj_id, cp_idx, pos):
            cp, cp_pos = c_points[cp_idx]
            if skip_resolved and (obj_id, cp) in self._resolved_events:
                return True
            return cp_pos <= pos if skip_immediate else cp_pos < pos

        cp_idx = 0
        events = []
        for (obj_id, pos) in obj_positions:
            assert pos >= 0 and pos <= self.length, \
                "`nextEvents` on conveyor with undefined object positions!"

            while cp_idx < len(c_points) and _skip_cond(obj_id, cp_idx, pos):
                cp_idx += 1

            if cp_idx < len(c_points):
                cp = c_points[cp_idx]
                obj = self.objects[obj_id]
                diff = (cp[1] - pos) / self.speed
                events.append((obj, cp[0], diff))

        events.sort(key=lambda p: p[2])
        return events

    def immediateEvents(self, obj=None, cps=None) -> List[Tuple[Any, Any, float]]:
        return [ev for ev in self.nextEvents(obj=obj, cps=cps, skip_immediate=False) if ev[2] == 0]

    def pickUnresolvedEvent(self) -> Union[Tuple[Any, Any, float], None]:
        assert self.resolving(), "picking event for resolution while not resolving"
        evs = self.nextEvents(skip_immediate=False)
        if len(evs) == 0:
            return None

        obj, cp, diff = evs[0]
        if diff > 0:
            return None

        self._resolved_events.add((obj.id, cp))
        # print('HEEEEEEEY {} {} {}'.format(self.model_id, obj, cp))
        return obj, cp, diff

    def resume(self):
        self._stateTransfer('resume')
        self._resume_time = self.env.time()

    def pause(self):
        self._stateTransfer('pause')
        time_diff = self.env.time() - self._resume_time
        assert time_diff >= 0, "Pause before resume??"
        self.skipTime(time_diff)

    def startResolving(self):
        self._stateTransfer('start_resolving')

    def endResolving(self):
        self._resolved_events = set()
        self._stateTransfer('end_resolving')

    def pristine(self):
        return self._state == 'pristine'

    def dirty(self):
        return self._state == 'dirty'

    def resolving(self):
        return self._state == 'resolving'

    def resolved(self):
        return self._state == 'resolved'

    def moving(self):
        return self._state == 'moving'

def all_next_events(models: Dict[int, ConveyorModel], **kwargs):
    res = []
    for conv_idx, model in models.items():
        evs = model.nextEvents(**kwargs)
        res = merge_sorted(res, [(conv_idx, ev) for ev in evs],
                           using=lambda p: p[1][2])
    return res

def all_unresolved_events(models: Dict[int, ConveyorModel]):
    while True:
        had_some = False
        for conv_idx, model in models.items():
            if model.dirty():
                model.startResolving()
            if model.resolving():
                ev = model.pickUnresolvedEvent()
                if ev is not None:
                    yield (conv_idx, ev)
                    had_some = True
        if not had_some:
            break

def find_max_speed(up_model, up_pos, up_speed, dist_to_end, max_speed, bag_radius=BAG_RADIUS):
    speed = max_speed
    while speed > 0:
        end_time = dist_to_end / speed
        up_nearest = up_model.nearestObject(up_pos, after=end_time, speed=up_speed)
        if up_nearest is None:
            break

        _, obj_pos = up_nearest
        if abs(up_pos - obj_pos) >= 2*bag_radius:
            break
        elif dist_to_end == 0:
            return 0
        speed = round(speed - SPEED_STEP, SPEED_ROUND_DIGITS)

    return max(0, speed)


import json
import db_query

def handle_message(bot, recipient_id, message):
    # Check if message is a quick reply
    quick_reply = message.get('quick_reply')
    if quick_reply:
        handle_quick_reply(bot, recipient_id, quick_reply)
        return

    # Welcome message
    resp = "WE ESCAPE xin chào, chúng tôi có thể giúp gì cho bạn?"

    quick_replies_payloads = [
        {
            "content_type": "text",
            "title": "Tìm game để chơi",
            "payload": json.dumps({ 'action' : 'find-games' })
        },
        {
            "content_type": "text",
            "title": "Xem các suất trống",
            "payload": json.dumps({ 'action' : 'find-available-slots' })
        },
        {
            "content_type": "text",
            "title": "Chat với nhân viên",
            "payload": json.dumps({ 'action' : 'cancel' })
        }
    ]

    if message.get('text'):
        bot.send_quick_reply_message(recipient_id=recipient_id, text=resp, quick_replies=quick_replies_payloads)

def handle_postback(bot, recipient_id, postback):
    # Get payload
    payload = json.loads(postback.get('payload'))
    print(recipient_id, payload)

def handle_quick_reply(bot, recipient_id, message):
    # Debug message
    print("Inside quick reply")

    # Get payload
    payload = json.loads(message.get('payload'))
    print(recipient_id, payload)

    action = payload.get('action')

    if action == 'find-games':
        print("Find Games")
        resp = "Bạn muốn chơi ở chi nhánh nào?"

        locations = db_query.get_location_by_city('Hanoi')

        quick_replies_payloads = []

        for location in locations:
            print(location)

            tmp_reply = {
                "content_type":"text",
                "title":location,
                "payload":json.dumps({
                    'action': 'find-games-by-location',
                    'location': location
                })
            }

            quick_replies_payloads.append(tmp_reply)

        print(bot.send_quick_reply_message(recipient_id=recipient_id, text=resp, quick_replies=quick_replies_payloads ))

    elif action == 'find-games-by-location':
        location = payload.get('location')

        print("Find games by location: ", location)
        resp = "Đây là các games có tại " + location + ":"

        games = db_query.get_games_by_location('Hanoi', location)

        for game in games:
            #print(game)
            resp += "\n" + game

        #print(resp)

        print(bot.send_text_message(recipient_id=recipient_id, message=resp))

    elif action == 'find-available-slots':
        print("Find Available Slots")
    elif action == 'cancel':
        print("Cancel")


from classes.robotUR import *
from main import robot_loop
import json

NAME = 'UR3-B'
ROBOT_IP = '10.40.30.11'

f = open(NAME + '.json',)
config = json.load(f)

SUBSCRIBE_TOPICS = []
PUBLISH_TOPICS = []

for i in config['SUBSCRIBE_TOPICS']:
    SUBSCRIBE_TOPICS.append(i)

for i in config['PUBLISH_TOPICS']:
    PUBLISH_TOPICS.append(i)

ROUTINES_PATH = 'routines/'+NAME+'_routines/'

robot = Robot(ROBOT_IP, NAME, 30004)
robot.connect_to_robot()
robot.start_rtde()
robot_loop(robot,NAME,SUBSCRIBE_TOPICS,PUBLISH_TOPICS,ROUTINES_PATH)


nos = [7, 8, 10, 3, 2, 4, 6]

# new evens list from nos list
# functional programming

ans = list(filter(lambda ele: not ele % 2, nos))
print(ans)

# squares of even cubes of odd
ans = tuple(map(lambda ele: ele ** 3 if ele % 2 else ele ** 2, nos))
print(ans)


from path_create_class import download_path
from gen_dl import open_webpage, download
from concurrent.futures import ThreadPoolExecutor as executor

class Image_Bam:
    def __init__(self, url):
        self.url = url
        self.path = download_path()
        self.open_webpage = open_webpage
        self.workers = 5

    def get_links(self):
        self.page = self.open_webpage(self.url, method = 'soup')
        self.soup = self.page.soup if self.page.good_response() else None
        self.links = [[item.get('src'), self.path.save_path] for item in self.soup.find_all('img') if item.get('src').find('imagebam') != -1]

# needs work, specifically, selenium to access the click through to modify links for download
    def get_images(self):
        with executor(max_workers = self.workers) as exe:
            self.results = exe.map(download, self.links)


#naive diagonal accessing

class Solution:
    def diagonalSum(self, mat: List[List[int]]) -> int:
        n=len(mat)
        sums=0
        for i in range(n):
            for j in range(n):
                if(i==j):
                    sums+=mat[i][j]
                elif(i+j==n-1 and i!=j):
                    sums+=mat[i][j]
        return sums


import logging

from google.appengine.api import memcache
from dataviz.restclient import RestClient

URL = "api.geonames.org"
USERNAME = ""

geonames = RestClient(URL, "")

def locate(name):
    city = memcache.get(name)
    logging.debug("Memcache has %r for key %r" % (city, name))
    if city is None:
        logging.warn("Cache miss for geonames, key %r" % name)
        response = geonames.get("searchJSON", q=name, username=USERNAME,
                                maxRows=1)
        city = {}
        if "totalResultsCount" in response and response['totalResultsCount']:
            city = {
                'name': response['geonames'][0]['toponymName'],
                'coord': [
                    response['geonames'][0]['lng'],
                    response['geonames'][0]['lat']
                ],
            }
        result = memcache.add(name, city)
        logging.debug("Memcached %r with key %r: %r" % (city, name, result))
    return city


# -*- coding: utf-8 -*-
# pylint: skip-file
# by Roman Golev

# Imports
from pyrevit import HOST_APP
from Autodesk.Revit.ApplicationServices import LanguageType
from pyrevit import script
from pyrevit import output
import os.path as op
import core

# Handling button
__doc__ = """Информация о проекте, программе и расширении / Basic Information about current project and extension"""
__title__ = "Info"
__author__ = "Roman Golev"
__context__ = 'zero-doc'

# Handle Language versions
if HOST_APP.language == LanguageType.Russian:
    user = "Имя пользователя"
    Rvers = "Версия Ревит"
    Bvers = "Версия расширения"
    lang = "RU"
else:
    user = "Username"
    Rvers = "Revit Version"
    Bvers = "Extension version"
    lang = "EN"
dir_path = op.dirname(op.realpath(__file__))
style = 'img {max-width: 589px; padding: 25px 0} span {display: block; text-align: center;}'
output.get_output().add_style(style)
output.get_output().set_width(500)
output.get_output().set_height(500)
output.get_output().center()
out = script.get_output()
out.print_html('
 pyArchitect Extension Tools for pyRevit
')

print(str(user) + ' : {}'.format(HOST_APP.username))
print(str(Rvers) + ' : {}'.format(HOST_APP.version_name + HOST_APP.build))
print(str(Rvers) + ' : {}'.format(HOST_APP.subversion))
print(str(Bvers) + ' : {}'.format(core.get_local_version()))

out.print_html('pyArchitect repository')
out.print_html('Roman Golev')


# -*- coding:utf-8 -*-

class Solution:
    def countDigitOne(self, n: int) -> int:
        digit, res = 1, 0

        # 2304; high = 23, cur = 0, low = 4
        high, cur, low = n // 10, n % 10, 0
        while high != 0 or cur != 0:
            print(cur)
            if cur == 0:
                res += high * digit
            elif cur == 1:
                res += high * digit + low + 1
            else:
                res += (high + 1) * digit

            low += (cur * digit)
            cur = high % 10
            high = high // 10
            digit = digit * 10
        return res

if __name__ == '__main__':
    n = 2304
    ans = Solution().countDigitOne(n)
    print(ans)

43.1～n整数中1出现的次数.py


from Application.Api.Domain.PlexUser import PlexUser
from Application.Api.Domain.Vote import Vote
from typing import List

class Movie(object):
    def __init__(self,
                 movie_id: int,
                 imdb_id: str,
                 movie_name: str,
                 creator: PlexUser,
                 approved: bool,
                 declined: bool,
                 deleted: bool,
                 private: bool,
                 channel_id: int,
                 trakt_id: int,
                 votes: List[Vote]):
        self.movie_id: int = movie_id
        self.imdb_id: str = imdb_id
        self.movie_name: str = movie_name
        self.creator: PlexUser = creator
        self.approved: bool = approved
        self.declined: bool = declined
        self.deleted: bool = deleted
        self.private: bool = private
        self.channel_id: int = channel_id
        self.trakt_id: int = trakt_id
        self.votes: List[Vote] = votes


import pygame
from pygame_constants import *

class GameButton:

    # Initiate button: location, text, size, locked or not
    def __init__(self, location, title, lock=False):
        self.location = location
        self.size = (location.X_END - location.X, location.Y_END - location.Y)
        self.title = title
        self.title_center = ((location.X_END + location.X) // 2, (location.Y_END + location.Y) // 2)
        self.backgroundTile = pygame.Surface(self.size)
        if lock:
            self.locked = False
            self.lockButton()
        else:
            self.locked = True
            self.unlockButton()

    def lockButton(self):
        if not self.locked:
            self.title_view = FONT_COURIER_20.render(self.title, True, WHITE, GREY)
            self.title_rect = self.title_view.get_rect()
            self.title_rect.center = self.title_center
            self.backgroundTile.fill(GREY)
            self.locked = True

    def unlockButton(self):
        if self.locked:
            self.title_view = FONT_COURIER_20.render(self.title, True, BLACK, ORANGE)
            self.title_rect = self.title_view.get_rect()
            self.title_rect.center = self.title_center
            self.backgroundTile.fill(ORANGE)
            self.locked = False

    # Draw the button
    def blitButton(self, screen):
        screen.blit(self.backgroundTile, (self.location.X, self.location.Y))
        screen.blit(self.title_view, self.title_rect)

    def isLocked(self):
        return self.locked


N = int(input())
dp = [0]*N
dp[0] = 1
if N == 1:
    print(dp[0])
    exit()
elif N == 2:
    dp[1] = 3
    print(dp[1])
    exit()
else:
    dp[1] = 3
    for i in range(2, N):
        dp[i] = (dp[i-1] + 2*dp[i-2]) % 10007
    print(dp[N-1])


# -*- coding: utf-8 -*-
"""
Created on Tue Jun  1 19:12:47 2021

@author: gama0
"""

import requests
from bs4 import BeautifulSoup
import time
from tqdm import tqdm
from fake_useragent import UserAgent
import copy
import os
# import numpy as np

# a list of textures to NOT include
filters = ['AcousticFoam',
  'Candy',
  'ChristmasTreeOrnament',
  'Facade',
  'Fence',
  'Fingerprint',
  'Foam',
  'Metal Walkway',
  'OfficeCeiling',
  'Pathway',
  'Paint00',
  'Painting',
  'PineNeedles',
  'Porcelain',
  'Road',
  'RockBrush',
  'Rust001',
  'Sign',
  'Scratches',
  'Smear',
  'Sticker',
  'Tape',
  'TreeEnd',
  'SurfaceImperfections',
  'Footsteps',
  'Substance']

# ambientCG website:
output_path = '../dataset/zip/'
home = 'https://ambientcg.com/'
url = "https://ambientcg.com/list?category=&type=Material&sort=Alphabet"

if not os.path.exists(output_path):
    os.makedirs(output_path)

#%%
def main():
    # agent to request
    user_agent = UserAgent()
    download_url = []

    method_url = [url + '&method=PBRMultiAngle']

    # for methods with multiple pages of content (1 page has 180 links)
    offsets = [0, 180, 360, 540, 720, 900]
    for offset in offsets:
        method_url.append(url + '&method=PBRApproximated' + '&offset={}'.format(offset))
        method_url.append(url + '&method=PBRPhotogrammetry' + '&offset={}'.format(offset))
        method_url.append(url + '&method=PBRProcedural' + '&offset={}'.format(offset))

    #%%
    # =========================== request the list to "download_url"===================
    for i in method_url:

        re = requests.get(i, headers = { 'user-agent': user_agent.random })
        soup = BeautifulSoup(re.text,'html.parser')

        elems = soup.find_all('a')
        for elem in elems:
            # texture_url + elem = each download url
            if elem.get('href').startswith("/view?id="):
                n = elem.get('href').split('/')[-1]
                download_url.append(home + n)

    copy_download_url = copy.deepcopy(download_url)

    # =========================== filters ===================
    # ( I don't know the reason why there's 99 elements need to be removed, but it can't do it completely at once.)
    n = 0
    for _ in range(5):
        for i in copy_download_url:
            for word in filters:
                if i.split('=')[-1].startswith(word):
                    # print('remove this!', i)
                    copy_download_url.remove(i)
                    n += 1

    copy_download_url.sort()
    # np.savetxt('test.txt',copy_download_url,delimiter="\n", fmt="%s")

    #%%
    # =========================== request download url ===================
    counts = 0
    for i in tqdm(range(len(copy_download_url))):
    # ==========================
        # 如果爬的過程斷掉，從多少開始...
        if i<counts:
            # counts += 1
            continue
    # ==========================
        # print(copy_download_url[i])
        re = requests.get(copy_download_url[i], # https://ambientcg.com/view?id=Tiles098
                        headers = { 'user-agent': user_agent.random })
        soup = BeautifulSoup(re.text,'html.parser')

        # if you want to choose png or JPG
        zip_url = []
        elems = soup.find_all('a', class_= "TextLink")
        for elem in elems:
            zip_url.append(elem.get('href'))

        r = requests.get(zip_url[0], # https://ambientcg.com/get?file=Tiles098_1K-JPG.zip
                        headers = { 'user-agent': user_agent.random })

        with open(output_path + zip_url[0].split('=')[-1], "wb") as zipfile:
            zipfile.write(r.content)

        time.sleep(0.1)
        counts += 1

if __name__ == "__main__":
    main()


__author__ = "Sebastian Heinlein <devel@glatzor.de>"
__all__ = ("get_tests_dir", "Chroot", "AptDaemonTestCase")

import inspect
import os
import shutil
import subprocess
import sys
import time
import tempfile
import unittest

if sys.version_info.major > 2:
    from http.server import HTTPServer
    from http.server import SimpleHTTPRequestHandler as HTTPRequestHandler
else:
    from BaseHTTPServer import HTTPServer
    from SimpleHTTPServer import SimpleHTTPRequestHandler as HTTPRequestHandler

import apt_pkg
import apt.auth

PY3K = sys.version_info.major > 2

class MockQueue(object):

    """A fake TransactionQueue which only provides a limbo attribute."""

    def __init__(self):
        self.limbo = {}

class Chroot(object):

    """Provides a chroot which can be used by APT."""

    def __init__(self, prefix="tmp"):
        self.path = tempfile.mkdtemp(prefix)

    def setup(self):
        """Setup the chroot and modify the apt configuration."""
        for subdir in ["alternatives", "info", "parts", "updates", "triggers"]:
            path = os.path.join(self.path, "var", "lib", "dpkg", subdir)
            os.makedirs(path)
        for fname in ["status", "available"]:
            with open(os.path.join(self.path, "var", "lib", "dpkg", fname),
                      "w"):
                pass
        os.makedirs(os.path.join(self.path, "var/cache/apt/archives/partial"))
        os.makedirs(os.path.join(self.path, "var/lib/apt/lists"))
        os.makedirs(os.path.join(self.path, "var/lib/apt/lists/partial"))
        os.makedirs(os.path.join(self.path, "etc/apt/apt.conf.d"))
        os.makedirs(os.path.join(self.path, "etc/apt/sources.list.d"))
        os.makedirs(os.path.join(self.path, "etc/apt/preferences.d"))
        os.makedirs(os.path.join(self.path, "etc/apt/trusted.gpg.d"))
        os.makedirs(os.path.join(self.path, "var/log"))
        os.makedirs(os.path.join(self.path, "media"))

        # Make apt use the new chroot
        dpkg_wrapper = os.path.join(get_tests_dir(), "dpkg-wrapper.sh")
        apt_key_wrapper = os.path.join(get_tests_dir(), "fakeroot-apt-key")
        config_path = os.path.join(self.path, "etc/apt/apt.conf.d/10chroot")
        with open(config_path, "w") as cnf:
            cnf.write("Dir::Bin::DPkg %s;" % dpkg_wrapper)
            cnf.write("Dir::Bin::Apt-Key %s;" % apt_key_wrapper)
        apt_pkg.read_config_file(apt_pkg.config, config_path)
        apt_pkg.init_system()
        apt_pkg.config["Dir"] = self.path

    def remove(self):
        """Remove the files of the chroot."""
        apt_pkg.config.clear("Dir")
        apt_pkg.config.clear("Dir::State::Status")
        apt_pkg.init()
        shutil.rmtree(self.path)

    def add_trusted_key(self):
        """Add glatzor's key to the trusted ones."""
        apt.auth.add_key_from_file(os.path.join(get_tests_dir(),
                                                "repo/glatzor.gpg"))

    def install_debfile(self, path, force_depends=False):
        """Install a package file into the chroot."""
        cmd_list = ["fakeroot", "dpkg", "--root", self.path,
                    "--log=%s/var/log/dpkg.log" % self.path]
        if force_depends:
            cmd_list.append("--force-depends")
        cmd_list.extend(["--install", path])
        cmd = subprocess.Popen(cmd_list,
                               env={"PATH": "/sbin:/bin:/usr/bin:/usr/sbin"})
        cmd.communicate()

    def add_test_repository(self, copy_list=True, copy_sig=True):
        """Add the test repository to the to the chroot."""
        return self.add_repository(os.path.join(get_tests_dir(), "repo"),
                                   copy_list, copy_sig)

    def add_cdrom_repository(self):
        """Emulate a repository on removable device."""
        # Create the content of a fake cdrom
        media_path = os.path.join(self.path, "tmp/fake-cdrom")
        # The cdom gets identified by the info file
        os.makedirs(os.path.join(media_path, ".disk"))
        with open(os.path.join(media_path, ".disk/info"), "w") as info:
            info.write("This is a fake CDROM")
        # Copy the test repository "on" the cdrom
        shutil.copytree(os.path.join(get_tests_dir(), "repo"),
                        os.path.join(media_path, "repo"))

        # Call apt-cdrom add
        mount_point = self.mount_cdrom()
        os.system("apt-cdrom add -m -d %s "
                  "-o 'Debug::Acquire::cdrom'=True "
                  "-o 'Acquire::cdrom::AutoDetect'=False "
                  "-o 'Dir'=%s" % (mount_point, self.path))
        self.unmount_cdrom()

        config_path = os.path.join(self.path, "etc/apt/apt.conf.d/11cdrom")
        with open(config_path, "w") as cnf:
            cnf.write('Debug::Acquire::cdrom True;\n'
                      'Acquire::cdrom::AutoDetect False;\n'
                      'Acquire::cdrom::NoMount True;\n'
                      'Acquire::cdrom::mount "%s";' % mount_point)

    def mount_cdrom(self):
        """Copy the repo information to the CDROM mount point."""
        mount_point = os.path.join(self.path, "media/cdrom")
        os.symlink(os.path.join(self.path, "tmp/fake-cdrom"), mount_point)
        return mount_point

    def unmount_cdrom(self):
        """Remove all files from the mount point."""
        os.unlink(os.path.join(self.path, "media/cdrom"))

    def add_repository(self, path, copy_list=True, copy_sig=True):
        """Add a sources.list entry to the chroot."""
        # Add a sources list
        lst_path = os.path.join(self.path, "etc/apt/sources.list")
        with open(lst_path, "w") as lst_file:
            lst_file.write("deb file://%s ./ # Test repository" % path)
        if copy_list:
            filename = apt_pkg.uri_to_filename("file://%s/." % path)
            shutil.copy(os.path.join(path, "Packages"),
                        "%s/var/lib/apt/lists/%s_Packages" % (self.path,
                                                              filename))
            if os.path.exists(os.path.join(path, "Release")):
                shutil.copy(os.path.join(path, "Release"),
                            "%s/var/lib/apt/lists/%s_Release" % (self.path,
                                                                 filename))
            if copy_sig and os.path.exists(os.path.join(path, "Release.gpg")):
                shutil.copy(os.path.join(path, "Release.gpg"),
                            "%s/var/lib/apt/lists/%s_Release.gpg" % (self.path,
                                                                     filename))

class AptDaemonTestCase(unittest.TestCase):

    @classmethod
    def setupClass(cls):
        # Start with a clean APT configuration to remove changes
        # of previous tests
        for key in set([key.split("::")[0] for key in apt_pkg.config.keys()]):
            apt_pkg.config.clear(key)
        apt_pkg.init_config()

    def start_fake_polkitd(self, options="all"):
        """Start a fake PolicyKit daemon.

        :param allowed_actions: comma separated list of allowed actions.
                                Defaults to all
        """
        try:
            env = os.environ.copy()
            env["DBUS_SYSTEM_BUS_ADDRESS"] = self.dbus_address
        except AttributeError:
            env = None
        dir = get_tests_dir()
        proc = subprocess.Popen([os.path.join(dir, "fake-polkitd.py"),
                                "--allowed-actions", options],
                                env=env)
        self.addCleanup(self._kill_process, proc)
        return proc

    def start_session_aptd(self, chroot="/", debug=True):
        """Start an aptdaemon which listens on the session D-Bus.

        :param chroot: path to the chroot
        """
        env = os.environ.copy()
        try:
            env["DBUS_SYSTEM_BUS_ADDRESS"] = self.dbus_address
        except AttributeError:
            pass
        try:
            env.pop("http_proxy")  # Unset a local proxy, see LP #1050799
        except KeyError:
            pass
        dir = get_tests_dir()
        if dir == "/usr/share/aptdaemon/tests":
            path = "/usr/sbin/aptd"
        else:
            path = os.path.join(dir, "../aptd")
        cmd = ["python3", path, "--disable-plugins",
               "--chroot", chroot]
        if debug:
            cmd.append("--debug")
        proc = subprocess.Popen(cmd, env=env)
        self.addCleanup(self._kill_process, proc)
        return proc

    def start_dbus_daemon(self):
        """Start a private D-Bus daemon and return its process and address."""
        proc, dbus_address = start_dbus_daemon()
        self.addCleanup(self._kill_process, proc)
        self.dbus_address = dbus_address

    def start_keyserver(self, filename=None):
        """Start a fake keyserver on hkp://localhost:19191

        Keyword arguments:
        filename -- Optional path to a GnuPG pulic key file which should
            be returned by lookups. By default the key of the test repo
            is used.
        """
        dir = tempfile.mkdtemp(prefix="keyserver-test-")
        self.addCleanup(shutil.rmtree, dir)
        os.mkdir(os.path.join(dir, "pks"))
        if filename is None:
            filename = os.path.join(get_tests_dir(), "repo/glatzor.gpg")
        shutil.copy(filename, os.path.join(dir, "pks", "lookup"))

        pid = os.fork()
        if pid == 0:
            # quiesce server log
            os.dup2(os.open('/dev/null', os.O_WRONLY), sys.stderr.fileno())
            os.chdir(dir)
            httpd = HTTPServer(('localhost', 19191), HTTPRequestHandler)
            httpd.serve_forever()
            os._exit(0)
        else:
            self.addCleanup(self._kill_pid, pid)

        # wait a bit until server is ready
        time.sleep(0.5)

    def _kill_pid(self, pid):
        os.kill(pid, 15)
        os.wait()

    def _kill_process(self, proc):
        proc.kill()
        proc.wait()

def get_tests_dir():
    """Return the absolute path to the tests directory."""
    # Try to detect a relative tests dir if we are running from the source
    # directory
    try:
        path = inspect.getsourcefile(sys.modules["aptdaemon.test"])
    except KeyError:
        path = inspect.getsourcefile(inspect.currentframe())
    path = os.path.realpath(path)
    relative_path = os.path.join(os.path.dirname(path), "../tests")
    if os.path.exists(os.path.join(relative_path, "repo/Packages")):
        return os.path.normpath(relative_path)
    # Fallback to an absolute path
    elif os.path.exists("/usr/share/aptdaemon/tests/repo/Packages"):
        return "/usr/share/aptdaemon/tests"
    else:
        raise Exception("Could not find tests direcotry")

def start_dbus_daemon():
    """Start a private D-Bus daemon and return its process and address."""
    config_path = os.path.join(get_tests_dir(), "dbus.conf")
    proc = subprocess.Popen(["dbus-daemon", "--nofork",
                             "--print-address", "--config-file",
                             config_path],
                            stdout=subprocess.PIPE)
    output = proc.stdout.readline().strip()
    if PY3K:
        dbus_address = output.decode()
    else:
        dbus_address = output
    return proc, dbus_address

# vim: ts=4 et sts=4


import numpy as np
import cupy as cp
from .layer import Layer
from regularisers import l2

class DenseLayer(Layer):

    def __init__(self, layer_name, incoming_chans=None, output_dim=None, with_bias=True,
                 weight_regulariser=None, weight_initialiser="normal"):
        super().__init__(layer_name)
        self.incoming_chans = incoming_chans
        self.output_dim = output_dim
        self.with_bias = with_bias
        self.weight_regulariser = weight_regulariser
        self.downstream_X = None
        self.weight_initialiser = weight_initialiser

        if incoming_chans is not None and output_dim is not None:
            if self.weight_initialiser == "glorot_uniform":
                limit = np.sqrt(6.0 / (self.incoming_chans + self.output_dim))
                weights = np.random.uniform(low=-limit,
                                            high=limit,
                                            size=(self.incoming_chans,
                                                self.output_dim)).astype(np.float32)
            elif self.weight_initialiser == "normal":
                weights = 0.01*np.random.randn(self.incoming_chans, self.output_dim).astype(np.float32)
            self.learned_params = {"weights": weights}
            self.grads = {"weights": np.zeros_like(weights)}

            if with_bias:
                bias = np.zeros(output_dim).astype(np.float32)
                self.learned_params.update({"bias": bias})
                self.grads.update({"bias": np.zeros_like(bias, dtype=np.float32)})
        else:
            self.learned_params = {}
            self.grads = {}

    def __repr__(self):
        return "DenseLayer({}, incoming_chans={}, output_dim={}, weight_regulariser={})".format(
                                       self.layer_name,
                                       self.incoming_chans,
                                       self.output_dim,
                                       repr(self.weight_regulariser)
                                       )

    def forward(self, X, test_mode=False):
        xp = cp.get_array_module(X)
        if not test_mode:
            self.downstream_X = X

        out = xp.dot(X, self.learned_params["weights"])
        if self.with_bias:
            out += self.learned_params["bias"][xp.newaxis, :]

        return out

    def backward(self, upstream_dx):
        xp = cp.get_array_module(upstream_dx)
        if self.with_bias:
            self.grads["bias"] = xp.sum(upstream_dx, axis=0)
        self.grads["weights"] = xp.dot(self.downstream_X.T, upstream_dx)
        if self.weight_regulariser:
            self.grads["weights"] += (
                self.weight_regulariser.backward(self.learned_params["weights"])
            )

        return xp.dot(upstream_dx, self.learned_params["weights"].T)

    def save_to_h5(self, open_f, save_grads=True):
        base_dset = open_f.create_dataset(self.layer_name + "/layer_info", dtype=np.float32)
        base_dset.attrs["type"] = self.__class__.__name__
        base_dset.attrs["incoming_chans"] = self.incoming_chans
        base_dset.attrs["output_dim"] = self.output_dim
        base_dset.attrs["with_bias"] = self.with_bias

        dset = open_f.create_dataset(self.layer_name + "/weights",
                                     self.learned_params['weights'].shape,
                                     dtype=self.learned_params['weights'].dtype)
        dset[:] = cp.asnumpy(self.learned_params['weights'])
        if self.weight_regulariser is not None:
            dset.attrs["weight_regulariser_type"] = np.string_(self.weight_regulariser.type)
            dset.attrs["weight_regulariser_strength"] = np.string_(self.weight_regulariser.strength)
        if self.with_bias:
            dset_bias = open_f.create_dataset(self.layer_name + "/bias",
                                              self.learned_params['bias'].shape,
                                              dtype=self.learned_params['bias'].dtype)
            dset_bias[:] = cp.asnumpy(self.learned_params['bias'])

        if save_grads:
            dset_grads = open_f.create_dataset(self.layer_name + "/grads/weights",
                                               self.grads['weights'].shape,
                                               dtype=self.learned_params['weights'].dtype)
            dset_grads[:] = cp.asnumpy(self.grads["weights"])
            if self.with_bias:
                dset_grads_bias = open_f.create_dataset(self.layer_name + "/grads/bias",
                                                        self.grads['bias'].shape,
                                                        dtype=self.learned_params['bias'].dtype)
                dset_grads_bias[:] = cp.asnumpy(self.grads["bias"])

    def load_from_h5(self, open_f, load_grads=True):
        self.incoming_chans = open_f[self.layer_name + '/layer_info'].attrs['incoming_chans']
        self.output_dim = open_f[self.layer_name + '/layer_info'].attrs['output_dim']
        self.with_bias = open_f[self.layer_name + '/layer_info'].attrs['with_bias']

        weight_regulariser_type = open_f[self.layer_name + '/weights'].attrs.get("weight_regulariser_type", None)
        if weight_regulariser_type:
            weight_regulariser_strength = open_f[self.layer_name + '/weights'].attrs["weight_regulariser_strength"]
            if weight_regulariser_type == b"l2":
                self.weight_regulariser = l2.l2(strength=float(weight_regulariser_strength))

        self.learned_params['weights'] = open_f[self.layer_name + '/weights'][:]
        if self.with_bias:
            self.learned_params['bias'] = open_f[self.layer_name + '/bias'][:]
        if load_grads:
            self.grads['weights'] = open_f[self.layer_name + '/grads/weights'][:]
            if self.with_bias:
                self.grads['bias'] = open_f[self.layer_name + '/grads/bias'][:]


from __future__ import unicode_literals
import frappe
from frappe import _
from frappe.utils import flt, getdate

def execute(filters=None):
	columns = get_columns()
	sl_entries = get_entries(filters)
	data = []

	for ri in sl_entries:
		data.append([ri.name, ri.customer, ri.shipping_instruction, ri.no_do,
		ri.container_no, ri.no_seal, ri.lisence_plate, ri.weight, ri.vendor_trucking,
		ri.received_do, ri.date, ri.tebus_bon_muat, ri.pic_tebus_bon, ri.pick_up_start,
		ri.pick_up_done, ri.pic_pick_up, ri.cetak_kartu_kuning, ri.pic_cetak_kartu,
		ri.gate_in, ri.pic_gate_in, ri.status_container
	])

	return columns, data

def get_columns():
	"""return columns"""

	columns = [
		_("No Job")+":Link/Rekap Export:150",
		_("Customer")+":Link/Customer:150",
		_("Shipping Instruction")+"::100",
		_("DO")+"::100",
		_("No Container")+"::100",
		_("No Seal")+"::100",
		_("No Mobil")+"::100",
		_("Weight")+":Float:100",
		_("Trucking")+"::100",
		_("Tgl Terima DO")+":Datetime:120",
		_("Tgl Stuffing")+":Datetime:120",
		_("Tebus Bon Muat")+"::100",
		_("PIC")+":Link/Employee:120",
		_("Tgl Pick Up Start")+":Datetime:120",
		_("Tgl Pick Up Done")+":Datetime:120",
		_("PIC")+":Link/Employee:120",
		_("Tgl Cetak Kartu Kuning")+":Datetime:120",
		_("PIC")+":Link/Employee:120",
		_("Tgl Gate In Pelabuhan")+":Datetime:120",
		_("PIC")+":Link/Employee:120",
		_("Status Container")+"::100",
	]

	return columns

def get_conditions(filters):
	conditions = ""
	if filters.get("no_job"):
		conditions += " and p1.`name` = '%s'" % frappe.db.escape(filters["no_job"])

	return conditions

def get_entries(filters):
	conditions = get_conditions(filters)
	return frappe.db.sql("""SELECT p1.name, p1.customer, p1.shipping_instruction,
	p1.no_do, p2.container_no, p2.no_seal, p2.license_plate, p2.weight,
	p2.vendor_trucking, p1.received_do, p2.date, p2.tebus_bon_muat, p2.pic_tebus_bon,
	p2.pick_up_start, p2.pick_up_done, p2.pic_pick_up, p2.cetak_kartu_kuning,
	p2.pic_cetak_kartu, p2.gate_in, p2.pic_gate_in, p2.status_container
	FROM `tabRekap Export` p1
	INNER JOIN `tabRekap Export Item` p2 ON p1.name = p2.parent
	WHERE p1.docstatus != '2' %s
	ORDER BY p1.`name` DESC""" %
		conditions, as_dict=1)