import torch
import numpy as np
import matplotlib.pyplot as plt
import torchmetrics
import torch.nn as nn
import torch.nn.functional as F

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

def process_image(img):
    tmp = img
    if tmp.shape[1] == 13:
        tmp = convert_to_segmentation(tmp)
    elif tmp.shape[1] == 4 or tmp.shape[1] == 1:
        tmp = tmp[:,:3,:,:].permute(0,2,3,1)
    return tmp.cpu()

def show_result(G, x_, y_, num_epoch):
    preds = process_image(G(x_))
    x = process_image(x_)
    y = process_image(y_)
    fig, ax = plt.subplots(x_.shape[0], 3, figsize=(6,10))
    for i in range(x_.shape[0]):
        ax[i, 0].get_xaxis().set_visible(False)
        ax[i, 0].get_yaxis().set_visible(False)
        ax[i, 1].get_xaxis().set_visible(False)
        ax[i, 1].get_yaxis().set_visible(False)
        ax[i, 2].get_xaxis().set_visible(False)
        ax[i, 2].get_yaxis().set_visible(False)
        ax[i, 0].cla()
        ax[i, 0].imshow(x[i])
        ax[i, 1].cla()
        ax[i, 1].imshow(preds[i])
        ax[i, 2].cla()
        ax[i, 2].imshow(torch.round(y[i] * 12))

    plt.tight_layout()
    label_epoch = 'Epoch {0}'.format(num_epoch)
    fig.text(0.5, 0, label_epoch, ha='center')
    label_input = 'Input'
    fig.text(0.18, 1, label_input, ha='center')
    label_output = 'Output'
    fig.text(0.5, 1, label_output, ha='center')
    label_truth = 'Ground truth'
    fig.text(0.81, 1, label_truth, ha='center')

    plt.show()

def convert_to_one_hot(labels, classes=13, scale_values=True):
    """
    Convert a segmentation image to one hot encoding

    @params
    labels: (batch_size, height, width) or (batch_size, 1, height, width)
    classes: number of classes
    scale_values: if True, labels is multiplied by classes then rounded to the nearest integer before converting to one hot encoding.

    @note
    labels must be integers between 0 and classes-1 unless scale_values is True.
    If scale_values is True, labels must be between 0 and 1,
    and will be multiplied by classes before being converted to one hot encoding.

    @return
    one_hot_images: (batch_size, num_classes, height, width)
    """

    if len(labels.shape) == 4:
        labels = labels.squeeze(1)

    if scale_values:
        labels = torch.round(labels * (classes-1))

    one_hot = torch.nn.functional.one_hot(labels.long(), num_classes=classes)
    one_hot = torch.permute(one_hot, [0, 3, 1, 2]).float()

    return one_hot

def convert_to_segmentation(one_hot_images):
    """
    Convert a one hot encoding image to segmentation image

    @params
    one_hot_images: (batch_size, num_classes, height, width)

    @return
    segmentation_images: (batch_size, 1, height, width)
    """
    return torch.argmax(one_hot_images, dim=1)

def meanIoU(pred, target, classes):
    """
    Give the mean IoU of the image and the mask

    @params
        img: (batch_size, classes, height, width)
        mask: (batch_size, classes, height, width)
        classes: number of classes

    @note it is required that the values in img and mask and intergers between 0 and classes-1

    @returns
        mean IoU
    """

    # convert pred to segmentation and back to one hot encoding
    pred = convert_to_one_hot(convert_to_segmentation(pred), classes=classes, scale_values=False)

    intersection = torch.sum(pred * target, dim=(2, 3))
    union = torch.sum(pred, dim=(2, 3)) + torch.sum(target, dim=(2, 3)) - intersection
    iou = (intersection + 1e-8) / (union + 1e-8)
    iou = torch.mean(iou, dim=1)
    return torch.mean(iou)

class DiceLoss(nn.Module):
    def __init__(self):
        super(DiceLoss, self).__init__()

    def forward(self, pred, target):
        pred = F.softmax(pred, dim=1)
        intersection = torch.sum(pred * target, dim=(1, 2, 3))
        union = torch.sum(pred, dim=(1, 2, 3)) + torch.sum(target, dim=(1, 2, 3))
        dice = (2 * intersection ) / (union + 1e-8)
        dice = torch.mean(1 - dice)
        return dice


from channels.generic.websocket import AsyncWebsocketConsumer
from dotenv import load_dotenv
from deepgram import Deepgram
from typing import Dict

import os

load_dotenv()

class TranscriptConsumer(AsyncWebsocketConsumer):
   dg_client = Deepgram(os.getenv('DEEPGRAM_API_KEY'))

   async def get_transcript(self, data: Dict) -> None:
       if 'channel' in data:
           transcript = data['channel']['alternatives'][0]['transcript']

           if transcript:
               await self.send(transcript)

   async def connect_to_deepgram(self):
       try:
           self.socket = await self.dg_client.transcription.live({'punctuate': True, 'interim_results': False})
           self.socket.registerHandler(self.socket.event.CLOSE, lambda c: print(f'Connection closed with code {c}.'))
           self.socket.registerHandler(self.socket.event.TRANSCRIPT_RECEIVED, self.get_transcript)

       except Exception as e:
           raise Exception(f'Could not open socket: {e}')

   async def connect(self):
       await self.connect_to_deepgram()
       await self.accept()

   async def disconnect(self, close_code):
       await self.channel_layer.group_discard(
           self.room_group_name,
           self.channel_name
       )

   async def receive(self, bytes_data):
       self.socket.send(bytes_data)


import numpy as np
from djitellopy import tello
import cv2
from time import sleep
from QRcodeFunc import *

def thresholding(img, hsvVals):
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    lower = np.array([hsvVals[0], hsvVals[1], hsvVals[2]])
    upper = np.array([hsvVals[3], hsvVals[4], hsvVals[5]])
    mask = cv2.inRange(hsv, lower, upper)
    return mask

def getContours(imgThres, img):
    cx, cy = 0,0
    # X = 0
    # Y = 0
    contours, hieracrhy = cv2.findContours(imgThres, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
    if len(contours) != 0:
        biggest = max(contours, key=cv2.contourArea)
        X= int(np.average(biggest[:,0,0]))
        Y=int(np.average(biggest[:,0,1]))
        cv2.drawContours(img, biggest, -1, (255, 0, 255), 7)
        cv2.circle(img, (X, Y), 10, (0, 0, 255), cv2.FILLED)
        # x, y, w, h = cv2.boundingRect(biggest)
        # cx = x + w // 2
        # cy = y + h // 2
        # cv2.drawContours(img, biggest, -1, (255, 0, 255), 7)
        # cv2.circle(img, (cx, cy), 10, (0, 255, 0), cv2.FILLED)
    return cx, cy
    # return X, Y

def getSensorOutput(imgThres, sensors):
    imgs = np.hsplit(imgThres, sensors)
    totalPixels = (img.shape[1] // sensors) * img.shape[0]
    senOut = []
    for x, im in enumerate(imgs):
        pixelCount = cv2.countNonZero(im)
        if pixelCount > threshold * totalPixels:
            senOut.append(1)
        else:
            senOut.append(0)
        cv2.imshow(str(x), im)
    # print(senOut)
    return senOut

def sendCommands(senOut, cx, cy):
    # rotateZ(cx, cy)

    global curve
    ## TRANSLATION
    lr = (cx - width // 2) // senstivity
    lr = int(np.clip(lr, -10, 10))
    if 2 > lr > -2: lr = 0
    ## Rotation
    if   senOut == [1, 0, 0]: curve = weights[0]
    elif senOut == [1, 1, 0]: curve = weights[1]
    elif senOut == [0, 1, 0]: curve = weights[2]
    elif senOut == [0, 1, 1]: curve = weights[3]
    elif senOut == [0, 0, 1]: curve = weights[4]
    elif senOut == [0, 0, 0]: curve = weights[2]
    elif senOut == [1, 1, 1]:
        curve = weights[2] # just like 1 0 0 but without fSpeed
        # me.send_rc_control(0, 0, 0, curve)
        # return 1

    elif senOut == [1, 0, 1]: curve = weights[2]
    me.send_rc_control(lr, fSpeed, 0, curve)

    # print("command sent")
    return 1

def rotateZ(cx,cy):
    height_half = height//2
    width_half = width//2
    if cy != height_half:
        slope = (cx - width_half) / (cy - height_half)
        theta = round((np.arctan(slope)) * (180 / np.pi))
        theta //= senstivity
        if cx < width_half and cy < height_half:
            me.rotate_counter_clockwise(theta)
        elif cx < width_half and cy > height_half:
            me.rotate_counter_clockwise(180+theta)
        elif cx > width_half and cy > height_half:
            me.rotate_clockwise(180-theta)
        elif cx > width_half and cy < height_half:
            me.rotate_clockwise(-1*theta)

def main():
    init()
    global me, cap, senstivity, fSpeed, curve, weights, hsvVals, height, width, threshold, img

    me = tello.Tello()
    me.connect()
    print(me.get_battery())
    me.streamon()
    # me.takeoff()
    # me.move_down(20)
    sleep(5)

    cap = cv2.VideoCapture(1)
    hsvVals =  [0,0,193,179,255,255]
    sensors = 3
    threshold = 0.20
    width, height = 480, 360
    senstivity = 3  # if number is high less sensitive
    weights = [-25, -15, 0, 15, 25]
    weights = [int(i*1) for i in weights]
    fSpeed = 15
    curve = 0
    qr_found = None
    i = 0
    while True:
        vals = getKeyboardInput(me, False)
        me.send_rc_control(vals[0], vals[1], vals[2], vals[3])  # LR, BF, DU, YV
        # _, img = cap.read()
        img = me.get_frame_read().frame
        img = cv2.resize(img, (width, height))
        img = cv2.flip(img, 0)

        # val = decoder(me, img, qr_found, fw=30)
        # if val !=  None:
        #     qr_found = val
        imgThres = thresholding(img, hsvVals)
        cx,cy = getContours(imgThres, img)  ## For Translation
        senOut = getSensorOutput(imgThres, sensors)  ## Rotation
        cv2.imshow("Path", imgThres)
        img = cv2.putText(img, str(me.get_battery()), (200, 200), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 0, 0), 2)
        cv2.imshow("Output", img)
        # if vals[4]:
            # print("command will be sent")
        sendCommands(senOut, cx, cy)
        cv2.waitKey(1)

if __name__ == "__main__":
    main()


#User function Template for python3

class Solution:
    def getSubstringWithEqual012(self, Str):
        # code here
        maps ={}
        c0 = 0
        c1 = 0
        c2 = 0
        maps[(0,0)] = 1
        ans = 0
        for i in range(0,len(Str)):
            if Str[i]=='0':
                c0 += 1
            elif Str[i]=='1':
                c1 += 1
            else:
                c2 += 1
            if maps.get((c1-c0,c2-c1))!=None:
                ans += maps.get((c1-c0,c2-c1))
                maps[(c1-c0,c2-c1)] += 1
            else:
                maps[(c1-c0,c2-c1)] = 1
        return ans

#{
 # Driver Code Starts
#Initial Template for Python 3

if __name__ == '__main__':

    t = int(input())

    for _ in range(t):
        Str = input()

        solObj = Solution()

        print(solObj.getSubstringWithEqual012(Str))

# } Driver Code Ends

GeeksforGeeks/Python3/Medium/Equal 0, 1 and 2/equal-0-1-and-2.py


from typing import *

def longestSuccessiveElements(a : List[int]) -> int:
    # Write your code here.
    a.sort()
    maxi = 1
    maxval = 0
    for i in range(len(a)-1):
        if a[i+1]-a[i]==1:
            maxi+=1
        elif (a[i]!=a[i+1]) :
            maxi =1
        maxval = max(maxi, maxval)
    return maxval

Array Questions/longest successive elements.py


#! /usr/bin/env python3

class list_custom_sort(list):
    def sort_insert(self):
        for i in range(1, len(self)):
            key = self[i]
            j = i - 1
            while j > -1 and self[j] > key:
                self[j + 1] = self[j]
                j = j - 1
            self[j + 1] = key

    def sort_merge(self):
        print(len(self))
        self._sort_merge(0, len(self))

    def _sort_merge(self, p, r):
        if r - p > 1:
            q = int((p + r) / 2)
            self._sort_merge(p, q)
            self._sort_merge(q, r)
            L = self[p : q]
            R = self[q : r]
            iL = 0
            iR = 0
            L = L + [10000]
            R = R + [10000]
            for i in range(p, r):
                if L[iL] < R[iR]:
                    self[i] = L[iL]
                    iL += 1
                else:
                    self[i] = R[iR]
                    iR += 1

if __name__ == "__main__":
    x = list_custom_sort([2,4,5,7,1,2,3,6])
    x.sort_merge()
    print(x)


def isPrime(n):
    if n <= 1: return False
    for i in range(2, n):
        if n % i == 0:
            return False;

    return True

def returnPrime(list):
    primes = []
    for l in list:
       for p in l:
           if isPrime(p):
              primes += p
    return primes


from __future__ import annotations

import logging
import time
from collections import Counter
from copy import deepcopy
from dataclasses import dataclass, field
from math import isclose
from typing import List, Optional

import numpy as np

from cctbx import uctbx
from iotbx import mtz
from libtbx import env
from rstbx.cftbx.coordinate_frame_helpers import align_reference_frame
from scitbx import matrix
from scitbx.math import r3_rotation_axis_and_angle_from_matrix

import dials.util.ext
from dials.algorithms.scaling.scaling_library import (
    MergedHalfDatasets,
    determine_best_unit_cell,
)
from dials.array_family import flex
from dials.util.batch_handling import (
    assign_batches_to_reflections,
    calculate_batch_offsets,
    get_image_ranges,
)
from dials.util.filter_reflections import filter_reflection_table
from dials.util.multi_dataset_handling import (
    assign_unique_identifiers,
    parse_multiple_datasets,
)
from dials.util.version import dials_version

logger = logging.getLogger(__name__)

class MTZWriterBase:
    """Helper for adding metadata, crystals and datasets to an mtz file object."""

    def __init__(self, space_group, unit_cell=None):
        """If a unit cell is provided, will be used as default unless specified
        for each crystal."""
        mtz_file = mtz.object()
        mtz_file.set_title(f"From {env.dispatcher_name}")
        date_str = time.strftime("%Y-%m-%d at %H:%M:%S %Z")
        if time.strftime("%Z") != "GMT":
            date_str += time.strftime("  (%Y-%m-%d at %H:%M:%S %Z)", time.gmtime())
        mtz_file.add_history(f"From {dials_version()}, run on {date_str}")
        mtz_file.set_space_group_info(space_group.info())
        self.mtz_file = mtz_file
        if unit_cell:
            self.unit_cell = unit_cell
        self.current_crystal = None
        self.current_dataset = None
        self.n_crystals = 0
        self.n_datasets = 0

    def add_crystal(self, crystal_name=None, project_name=None, unit_cell=None):
        """Add a crystal to the mtz file object."""
        if not unit_cell:
            if not self.unit_cell:
                raise ValueError("Unit cell must be provided.")
            else:
                unit_cell = self.unit_cell
        if not crystal_name:
            crystal_name = f"crystal_{self.n_crystals + 1}"
        if not project_name:
            project_name = "DIALS"
        self.current_crystal = self.mtz_file.add_crystal(
            crystal_name, project_name, unit_cell.parameters()
        )
        self.n_crystals += 1

    def add_empty_dataset(self, wavelength, name=None):
        """Add an empty dataset object to the mtz file."""
        if not name:
            name = "FROMDIALS"
        self.current_dataset = self.current_crystal.add_dataset(name, wavelength)
        self.n_datasets += 1

class MergedMTZWriter(MTZWriterBase):
    """Mtz writer for merged data."""

    def add_dataset(
        self,
        merged_array=None,
        anom_array=None,
        amplitudes=None,
        anom_amplitudes=None,
        dano=None,
        multiplicities=None,
        anom_multiplicities=None,
        suffix=None,
        half_datasets: Optional[MergedHalfDatasets] = None,
        r_free_array=None,
    ):
        """Add merged data to the most recent dataset.

        Args:
            merged_array: A merged miller array of IMEAN intensities
            wavelength: The wavelength of the dataset
            anom_array (Optional): An anomalous merged miller array
            amplitudes (Optional): A merged miller array of amplitudes
            anom_amplitudes (Optional): An anomalous merged array of amplitudes
            suffix (Optional[str]): Column name suffix to use for this dataset.
        """
        if not suffix:
            suffix = ""
        if merged_array:
            self.current_dataset.add_miller_array(merged_array, "IMEAN" + suffix)
        if multiplicities:
            self.current_dataset.add_miller_array(multiplicities, "N" + suffix)
        if amplitudes:
            self.current_dataset.add_miller_array(amplitudes, "F" + suffix)
        if anom_array:
            self.current_dataset.add_miller_array(anom_array, "I" + suffix)
        if anom_multiplicities:
            self.current_dataset.add_miller_array(anom_multiplicities, "N" + suffix)
        if anom_amplitudes:
            self.current_dataset.add_miller_array(anom_amplitudes, "F" + suffix)
        if dano:
            self.current_dataset.add_miller_array(
                dano, "DANO" + suffix, column_types="DQ"
            )
        if half_datasets:
            self.current_dataset.add_miller_array(
                half_datasets.data1, "IHALF1" + suffix, column_types="JQ"
            )
            self.current_dataset.add_miller_array(
                half_datasets.data2, "IHALF2" + suffix, column_types="JQ"
            )
            self.current_dataset.add_miller_array(
                half_datasets.multiplicity1,
                "NHALF1" + suffix,
            )
            self.current_dataset.add_miller_array(
                half_datasets.multiplicity2,
                "NHALF2" + suffix,
            )
        if r_free_array:
            self.current_dataset.add_miller_array(
                r_free_array, column_root_label="FreeR_flag", column_types="I"
            )

class MADMergedMTZWriter(MergedMTZWriter):
    """Mtz writer for multi-wavelength merged data."""

    def add_dataset(
        self,
        merged_array=None,
        anom_array=None,
        amplitudes=None,
        anom_amplitudes=None,
        dano=None,
        multiplicities=None,
        anom_multiplicities=None,
        suffix=None,
        half_datasets: Optional[MergedHalfDatasets] = None,
        r_free_array=None,
    ):
        if not suffix:
            suffix = f"_WAVE{self.n_datasets}"
        super().add_dataset(
            merged_array,
            anom_array,
            amplitudes,
            anom_amplitudes,
            dano,
            multiplicities,
            anom_multiplicities,
            suffix,
            half_datasets,
            r_free_array=r_free_array,
        )

class UnmergedMTZWriter(MTZWriterBase):
    def add_batch_list(
        self,
        image_range,
        experiment,
        wavelength,
        dataset_id,
        batch_offset,
        force_static_model,
    ):
        """Add batch metadata to the mtz file."""

        # Recalculate useful numbers and references here
        n_batches = image_range[1] - image_range[0] + 1
        phi_start = flex.float(n_batches, 0)
        phi_range = flex.float(n_batches, 0)
        umat_array = flex.float(flex.grid(n_batches, 9))
        cell_array = flex.float(flex.grid(n_batches, 6))

        # Reciprocal lattice vectors in the lab frame at zero scan angle
        if experiment.goniometer:
            S = matrix.sqr(experiment.goniometer.get_setting_rotation())
            F = matrix.sqr(experiment.goniometer.get_fixed_rotation())
            UBlab = S * F * matrix.sqr(experiment.crystal.get_A())

            axis = matrix.col(experiment.goniometer.get_rotation_axis())
            axis_datum = matrix.col(experiment.goniometer.get_rotation_axis_datum())

        else:
            UBlab = matrix.sqr(experiment.crystal.get_A())

        i0 = image_range[0]
        for i in range(n_batches):
            if experiment.scan:
                phi_start[i], phi_range[i] = experiment.scan.get_image_oscillation(
                    i + i0
                )

            # Unit cell and UB matrix for the centre of the image for scan-varying model
            if (
                not force_static_model
                and experiment.crystal.num_scan_points > 0
                and experiment.goniometer
            ):

                # Get the index of the image in the sequence e.g. first => 0, second => 1
                image_index = i + i0 - experiment.scan.get_image_range()[0]

                # Find the U matrix at the frame centre by calculating the linear transform
                # that goes from the start of the frame to the end, and then applying half of
                # that to the start value
                U0 = matrix.sqr(experiment.crystal.get_U_at_scan_point(image_index))
                U1 = matrix.sqr(experiment.crystal.get_U_at_scan_point(image_index + 1))
                M = U1 * U0.inverse()
                (
                    angle_M,
                    axis_M,
                ) = M.r3_rotation_matrix_as_unit_quaternion().unit_quaternion_as_axis_and_angle(
                    deg=False
                )
                M_half = axis_M.axis_and_angle_as_r3_rotation_matrix(
                    angle_M / 2, deg=False
                )
                Ucentre = M_half * U0

                # Find the B matrix at the frame centre by interpolation
                B0 = matrix.sqr(experiment.crystal.get_B_at_scan_point(image_index))
                B1 = matrix.sqr(experiment.crystal.get_B_at_scan_point(image_index + 1))
                Bcentre = (B0 + B1) / 2

                # Unit cell at the frame centre
                unit_cell = uctbx.unit_cell(
                    orthogonalization_matrix=Bcentre.transpose().inverse()
                )

                # Get full lab frame UB then unwind to zero scan angle
                phi_centre = phi_start[i] + phi_range[i] / 2
                R = matrix.sqr(
                    axis_datum.axis_and_angle_as_r3_rotation_matrix(
                        phi_centre, deg=False
                    )
                )
                Rlab_inv = matrix.sqr(
                    axis.axis_and_angle_as_r3_rotation_matrix(-phi_centre, deg=False)
                )
                _UBlab = Rlab_inv * S * R * F * Ucentre * Bcentre

            else:
                unit_cell = experiment.crystal.get_unit_cell()
                _UBlab = UBlab

            # We assume a single-axis goniometer as it is not clear that multi-
            # axis goniometry was ever fully supported in MTZ format. Orientation
            # will be taken from the laboratory frame for this image.
            U = matrix.sqr(dials.util.ext.ub_to_mosflm_u(_UBlab, unit_cell))

            # FIXME need to get what was refined and what was constrained from the
            # crystal model - see https://github.com/dials/dials/issues/355
            _unit_cell_params = unit_cell.parameters()
            for j in range(6):
                cell_array[i, j] = _unit_cell_params[j]
            # Transpose to put in column-major order for MTZ export
            U_t_elements = U.transpose().elems
            for j in range(9):
                umat_array[i, j] = U_t_elements[j]

        # We ignore panels beyond the first one, at the moment
        panel = experiment.detector[0]
        panel_size = panel.get_image_size()
        panel_distance = panel.get_directed_distance()

        if experiment.goniometer:
            axis = flex.float(experiment.goniometer.get_rotation_axis())
        else:
            axis = flex.float((0.0, 0.0, 0.0))

        source = flex.float(experiment.beam.get_sample_to_source_direction())

        # get the mosaic spread though today it may not actually be set - should
        # this be in the BATCH headers?
        try:
            mosaic = experiment.crystal.get_mosaicity()
        except AttributeError:
            mosaic = 0.0

        # Jump into C++ to do the rest of the work
        dials.util.ext.add_dials_batches(
            self.mtz_file,
            dataset_id,
            image_range,
            batch_offset,
            wavelength,
            mosaic,
            phi_start,
            phi_range,
            cell_array,
            umat_array,
            panel_size,
            panel_distance,
            axis,
            source,
        )

    def write_columns(self, reflection_table):
        """Write the column definitions AND data to the current dataset."""

        # now create the actual data structures - first keep a track of the columns

        # H K L M/ISYM BATCH I SIGI IPR SIGIPR FRACTIONCALC XDET YDET ROT WIDTH
        # LP MPART FLAG BGPKRATIOS

        # gather the required information for the reflection file

        nref = len(reflection_table["miller_index"])
        assert nref
        xdet, ydet, _ = [
            flex.double(x) for x in reflection_table["xyzobs.px.value"].parts()
        ]

        # now add column information...

        # FIXME add DIALS_FLAG which can include e.g. was partial etc.

        type_table = {
            "H": "H",
            "K": "H",
            "L": "H",
            "I": "J",
            "SIGI": "Q",
            "IPR": "J",
            "SIGIPR": "Q",
            "BG": "R",
            "SIGBG": "R",
            "XDET": "R",
            "YDET": "R",
            "BATCH": "B",
            "BGPKRATIOS": "R",
            "WIDTH": "R",
            "MPART": "I",
            "M_ISYM": "Y",
            "FLAG": "I",
            "LP": "R",
            "FRACTIONCALC": "R",
            "ROT": "R",
            "QE": "R",
        }

        # derive index columns from original indices with
        #
        # from m.replace_original_index_miller_indices
        #
        # so all that is needed now is to make space for the reflections - fill with
        # zeros...

        self.mtz_file.adjust_column_array_sizes(nref)
        self.mtz_file.set_n_reflections(nref)
        dataset = self.current_dataset

        # assign H, K, L, M_ISYM space
        for column in "H", "K", "L", "M_ISYM":
            dataset.add_column(column, type_table[column]).set_values(
                flex.double(nref, 0.0).as_float()
            )

        self.mtz_file.replace_original_index_miller_indices(
            reflection_table["miller_index"]
        )

        dataset.add_column("BATCH", type_table["BATCH"]).set_values(
            reflection_table["batch"].as_double().as_float()
        )

        # if intensity values used in scaling exist, then just export these as I, SIGI
        if "intensity.scale.value" in reflection_table:
            I_scaling = reflection_table["intensity.scale.value"]
            V_scaling = reflection_table["intensity.scale.variance"]
            # Trap negative variances
            assert V_scaling.all_gt(0)
            dataset.add_column("I", type_table["I"]).set_values(I_scaling.as_float())
            dataset.add_column("SIGI", type_table["SIGI"]).set_values(
                flex.sqrt(V_scaling).as_float()
            )
            dataset.add_column("SCALEUSED", "R").set_values(
                reflection_table["inverse_scale_factor"].as_float()
            )
            dataset.add_column("SIGSCALEUSED", "R").set_values(
                flex.sqrt(reflection_table["inverse_scale_factor_variance"]).as_float()
            )
        else:
            if "intensity.prf.value" in reflection_table:
                if "intensity.sum.value" in reflection_table:
                    col_names = ("IPR", "SIGIPR")
                else:
                    col_names = ("I", "SIGI")
                I_profile = reflection_table["intensity.prf.value"]
                V_profile = reflection_table["intensity.prf.variance"]
                # Trap negative variances
                assert V_profile.all_gt(0)
                dataset.add_column(col_names[0], type_table["I"]).set_values(
                    I_profile.as_float()
                )
                dataset.add_column(col_names[1], type_table["SIGI"]).set_values(
                    flex.sqrt(V_profile).as_float()
                )
            if "intensity.sum.value" in reflection_table:
                I_sum = reflection_table["intensity.sum.value"]
                V_sum = reflection_table["intensity.sum.variance"]
                # Trap negative variances
                assert V_sum.all_gt(0)
                dataset.add_column("I", type_table["I"]).set_values(I_sum.as_float())
                dataset.add_column("SIGI", type_table["SIGI"]).set_values(
                    flex.sqrt(V_sum).as_float()
                )
        if (
            "background.sum.value" in reflection_table
            and "background.sum.variance" in reflection_table
        ):
            bg = reflection_table["background.sum.value"]
            varbg = reflection_table["background.sum.variance"]
            assert (varbg >= 0).count(False) == 0
            sigbg = flex.sqrt(varbg)
            dataset.add_column("BG", type_table["BG"]).set_values(bg.as_float())
            dataset.add_column("SIGBG", type_table["SIGBG"]).set_values(
                sigbg.as_float()
            )

        dataset.add_column("FRACTIONCALC", type_table["FRACTIONCALC"]).set_values(
            reflection_table["fractioncalc"].as_float()
        )

        dataset.add_column("XDET", type_table["XDET"]).set_values(xdet.as_float())
        dataset.add_column("YDET", type_table["YDET"]).set_values(ydet.as_float())
        dataset.add_column("ROT", type_table["ROT"]).set_values(
            reflection_table["ROT"].as_float()
        )
        if "lp" in reflection_table:
            dataset.add_column("LP", type_table["LP"]).set_values(
                reflection_table["lp"].as_float()
            )
        if "qe" in reflection_table:
            dataset.add_column("QE", type_table["QE"]).set_values(
                reflection_table["qe"].as_float()
            )
        elif "dqe" in reflection_table:
            dataset.add_column("QE", type_table["QE"]).set_values(
                reflection_table["dqe"].as_float()
            )
        else:
            dataset.add_column("QE", type_table["QE"]).set_values(
                flex.double(nref, 1.0).as_float()
            )

def export_mtz(
    reflection_table,
    experiment_list,
    intensity_choice,
    filename,
    best_unit_cell=None,
    partiality_threshold=0.4,
    combine_partials=True,
    min_isigi=-5,
    filter_ice_rings=False,
    d_min=None,
    force_static_model=False,
    crystal_name=None,
    project_name=None,
    wavelength_tolerance=1e-4,
):
    """Export data from reflection_table corresponding to experiment_list to an
    MTZ file hklout."""

    # First get the experiment identifier information out of the data
    expids_in_table = reflection_table.experiment_identifiers()
    if not list(expids_in_table.keys()):
        reflection_tables = parse_multiple_datasets([reflection_table])
        experiment_list, refl_list = assign_unique_identifiers(
            experiment_list, reflection_tables
        )
        reflection_table = flex.reflection_table()
        for reflections in refl_list:
            reflection_table.extend(reflections)
        expids_in_table = reflection_table.experiment_identifiers()
    reflection_table.assert_experiment_identifiers_are_consistent(experiment_list)
    expids_in_list = list(experiment_list.identifiers())

    # Convert geometry to the Cambridge frame
    experiment_list = convert_to_cambridge(experiment_list)

    # Convert experiment_list to a real python list or else identity assumptions
    # fail like:
    #   assert experiment_list[0] is experiment_list[0]
    # And assumptions about added attributes break
    experiment_list = list(experiment_list)

    # Validate multi-experiment assumptions
    if len(experiment_list) > 1:
        # All experiments should match crystals, or else we need multiple crystals/datasets
        if not all(
            x.crystal == experiment_list[0].crystal for x in experiment_list[1:]
        ):
            logger.warning(
                "Experiment crystals differ. Using first experiment crystal for file-level data."
            )

        # At least, all experiments must have the same space group
        if len({x.crystal.get_space_group().make_tidy() for x in experiment_list}) != 1:
            raise ValueError("Experiments do not have a unique space group")

    wavelengths = match_wavelengths(experiment_list, wavelength_tolerance)
    for w in wavelengths.values():
        w.calculate_weighted_mean([reflection_table])

    if len(wavelengths) > 1:
        identifiers_list = [e.identifier for e in experiment_list]
        logger.info(
            "Multiple wavelengths found: \n%s",
            "\n".join(
                "  Wavlength: %.5f, experiment numbers: %s "
                % (
                    v.weighted_mean,
                    ",".join(
                        map(str, [identifiers_list.index(i) for i in v.identifiers])
                    ),
                )
                for v in wavelengths.values()
            ),
        )

    # also only work correctly with one panel (for the moment)
    if any(len(experiment.detector) != 1 for experiment in experiment_list):
        logger.warning("Ignoring multiple panels in output MTZ")

    if best_unit_cell is None:
        best_unit_cell = determine_best_unit_cell(experiment_list)
    reflection_table["d"] = best_unit_cell.d(reflection_table["miller_index"])

    # Clean up the data with the passed in options
    reflection_table = filter_reflection_table(
        reflection_table,
        intensity_choice=intensity_choice,
        partiality_threshold=partiality_threshold,
        combine_partials=combine_partials,
        min_isigi=min_isigi,
        filter_ice_rings=filter_ice_rings,
        d_min=d_min,
    )

    # get batch offsets and image ranges - even for scanless experiments
    batch_offsets = [
        expt.scan.get_batch_offset()
        for expt in experiment_list
        if expt.scan is not None
    ]
    unique_offsets = set(batch_offsets)
    if len(set(unique_offsets)) <= 1:
        logger.debug("Calculating new batches")
        batch_offsets = calculate_batch_offsets(experiment_list)
        batch_starts = [
            e.scan.get_image_range()[0] if e.scan else 0 for e in experiment_list
        ]
        effective_offsets = [o + s for o, s in zip(batch_offsets, batch_starts)]
        unique_offsets = set(effective_offsets)
    else:
        logger.debug("Keeping existing batches")
    image_ranges = get_image_ranges(experiment_list)
    if len(unique_offsets) != len(batch_offsets):

        raise ValueError(
            "Duplicate batch offsets detected: %s"
            % ", ".join(
                str(item) for item, count in Counter(batch_offsets).items() if count > 1
            )
        )

    # Create the mtz file
    mtz_writer = UnmergedMTZWriter(experiment_list[0].crystal.get_space_group())

    # FIXME TODO for more than one experiment into an MTZ file:
    #
    # - add an epoch (or recover an epoch) from the scan and add this as an extra
    #   column to the MTZ file for scaling, so we know that the two lattices were
    #   integrated at the same time
    # ✓ decide a sensible BATCH increment to apply to the BATCH value between
    #   experiments and add this

    for id_ in expids_in_table.keys():
        # Grab our subset of the data
        loc = expids_in_list.index(
            expids_in_table[id_]
        )  # get strid and use to find loc in list
        experiment = experiment_list[loc]
        identifier = experiment.identifier

        if len(wavelengths) > 1:
            for i, wl in enumerate(wavelengths.values()):
                if identifier in wl.identifiers:
                    wavelength = wl.weighted_mean
                    dataset_id = i + 1
                    break
        else:
            wavelength = list(wavelengths.values())[0].weighted_mean
            dataset_id = 1

        reflections = reflection_table.select(reflection_table["id"] == id_)
        batch_offset = batch_offsets[loc]
        image_range = image_ranges[loc]
        reflections = assign_batches_to_reflections([reflections], [batch_offset])[0]
        experiment.data = dict(reflections)

        s0n = matrix.col(experiment.beam.get_s0()).normalize().elems
        logger.debug("Beam vector: %.4f %.4f %.4f" % s0n)

        mtz_writer.add_batch_list(
            image_range,
            experiment,
            wavelength,
            dataset_id,
            batch_offset=batch_offset,
            force_static_model=force_static_model,
        )

        # Create the batch offset array. This gives us an experiment (id)-dependent
        # batch offset to calculate the correct batch from image number.
        experiment.data["batch_offset"] = flex.int(
            len(experiment.data["id"]), batch_offset
        )

        # Calculate whether we have a ROT value for this experiment, and set the column
        _, _, z = experiment.data["xyzcal.px"].parts()
        if experiment.scan:
            experiment.data["ROT"] = experiment.scan.get_angle_from_array_index(z)
        else:
            experiment.data["ROT"] = z

    mtz_writer.add_crystal(
        crystal_name=crystal_name,
        project_name=project_name,
        unit_cell=best_unit_cell,
    )
    # Note: add unit cell here as may have changed basis since creating mtz.
    # For multi-wave unmerged mtz, we add an empty dataset for each wavelength,
    # but only write the data into the final dataset (for unmerged the batches
    # link the unmerged data to the individual wavelengths).
    for wavelength in wavelengths.values():
        mtz_writer.add_empty_dataset(wavelength.weighted_mean)

    # Combine all of the experiment data columns before writing
    combined_data = {k: v.deep_copy() for k, v in experiment_list[0].data.items()}
    for experiment in experiment_list[1:]:
        for k, v in experiment.data.items():
            combined_data[k].extend(v)
    # ALL columns must be the same length
    assert len({len(v) for v in combined_data.values()}) == 1, "Column length mismatch"
    assert len(combined_data["id"]) == len(
        reflection_table["id"]
    ), "Lost rows in split/combine"

    # Write all the data and columns to the mtz file
    mtz_writer.write_columns(combined_data)

    logger.info(
        "Saving %s integrated reflections to %s", len(combined_data["id"]), filename
    )
    mtz_file = mtz_writer.mtz_file
    mtz_file.write(filename)

    return mtz_file

@dataclass
class WavelengthGroup:
    min_wl: float
    max_possible_wl: float
    identifiers: list[str] = field(default_factory=list)
    exp_nos: list[int] = field(default_factory=list)
    wavelengths: list[float] = field(default_factory=list)
    weighted_mean: float = 0

    def add_experiment(self, identifier: str, loc_in_list: int, wl: float) -> None:
        self.identifiers.append(identifier)
        self.exp_nos.append(loc_in_list)
        self.wavelengths.append(wl)

    def calculate_weighted_mean(
        self, reflection_tables: List[flex.reflection_table]
    ) -> None:
        n, nw = (0, 0)
        for i, w in zip(self.identifiers, self.wavelengths):
            for table in reflection_tables:
                refls = table.select_on_experiment_identifiers([i])
                n_this = refls.select(
                    refls.get_flags(refls.flags.integrated, all=False)
                ).size()
                if n_this:
                    n += n_this
                    nw += n_this * w
                    break
        if n:
            self.weighted_mean = nw / n

def match_wavelengths(experiments, absolute_tolerance=1e-4):
    wavelengths = {}
    for i, x in enumerate(experiments):
        w = x.beam.get_wavelength()
        matches = [isclose(w, k, abs_tol=absolute_tolerance) for k in wavelengths]
        if not any(matches):
            wavelengths[w] = WavelengthGroup(w, w + absolute_tolerance)
            wavelengths[w].add_experiment(x.identifier, i, w)
        else:
            match_w = list(wavelengths.keys())[matches.index(True)]
            wavelengths[match_w].add_experiment(x.identifier, i, w)
    return wavelengths

def convert_to_cambridge(experiments):
    """Rotate the geometry of an experiment list to match the Cambridge frame,
    in which X is along the idealized X-ray beam and Z is along the primary
    rotation axis"""

    # First handle the potential for shared experiment models - we don't
    # want to apply multiple transformations to shared models, simplest way is
    # to make a copy for each experiment.
    n_expt = len(experiments)
    if len(experiments.crystals()) < n_expt:
        for expt in experiments:
            expt.crystal = deepcopy(expt.crystal)
    if len(experiments.beams()) < n_expt:
        for expt in experiments:
            expt.beam = deepcopy(expt.beam)
    if len(experiments.detectors()) < n_expt:
        for expt in experiments:
            expt.detector = deepcopy(expt.detector)
    if any(experiments.goniometers()) and len(experiments.goniometers()) < n_expt:
        for expt in experiments:
            expt.goniometer = deepcopy(expt.goniometer)

    for expt in experiments:
        if expt.goniometer:
            primary_axis = matrix.col(expt.goniometer.get_rotation_axis_datum())
        else:
            primary_axis = matrix.col((1.0, 0.0, 0.0))
        us0 = expt.beam.get_unit_s0()

        R = align_reference_frame(primary_axis, (0, 0, 1), us0, (1, 0, 0))
        axis_angle = r3_rotation_axis_and_angle_from_matrix(R)
        axis = matrix.col(axis_angle.axis)
        angle = axis_angle.angle()
        logger.debug(
            f"Rotating experiment{'s' if len(experiments) else ''} about axis {axis.elems} by {np.degrees(angle):.2f}°"
        )
        expt.detector.rotate_around_origin(axis, angle, deg=False)
        expt.beam.rotate_around_origin(axis, angle, deg=False)

        # For the goniometer, each component needs transformation
        if expt.goniometer:
            F = matrix.sqr(expt.goniometer.get_fixed_rotation())
            expt.goniometer.set_fixed_rotation(R * F * R.transpose())
            expt.goniometer.set_rotation_axis_datum(R * primary_axis)
            S = matrix.sqr(expt.goniometer.get_setting_rotation())
            expt.goniometer.set_setting_rotation(R * S * R.transpose())

        if expt.crystal is not None:
            expt.crystal = rotate_crystal(expt.crystal, R, axis, angle)

    return experiments

def rotate_crystal(crystal, Rmat, axis, angle):

    Amats = []
    if crystal.num_scan_points > 0:
        scan_pts = list(range(crystal.num_scan_points))
        Amats = [
            Rmat
            * matrix.sqr(crystal.get_U_at_scan_point(t))
            * matrix.sqr(crystal.get_B_at_scan_point(t))
            for t in scan_pts
        ]

    crystal.rotate_around_origin(axis, angle, deg=False)
    if Amats:
        crystal.set_A_at_scan_points(Amats)

    return crystal


import pandas as pd
from tqdm import tqdm
import numpy as np
import gc
import multiprocessing as mlp
from multiprocessing import Manager
from itertools import combinations,product
from tool import co_prob,condition_prob,condition_stat,num2bin
from process_worker import prob_feature_worker,tfidf_worker,conti_stat_feature_worker
from sklearn.linear_model import Ridge
from keras.callbacks import EarlyStopping

"""
tem_id                  - Ad id.
user_id                 - User id.
region                  - Ad region.
city                    - Ad city.
parent_category_name    - Top level ad category as classified by Avito's ad model.
category_name           - Fine grain ad category as classified by Avito's ad model.
param_1                 - Optional parameter from Avito's ad model.
param_2                 - Optional parameter from Avito's ad model.
param_3                 - Optional parameter from Avito's ad model.
title                   - Ad title.
description             - Ad description.
price                   - Ad price.
item_seq_number         - Ad sequential number for user.
activation_date         - Date ad was placed.
user_type               - User type.
image                   - Id code of image. Ties to a jpg file in train_jpg. Not every ad has an image.
image_top_1             - Avito's classification code for the image.
"""

def baseFeature(df):
    df['weekday'] = df.activation_date.dt.weekday
    df['month'] = df.activation_date.dt.month
    df['day'] = df.activation_date.dt.day
    df['week'] = df.activation_date.dt.week

    # length of description
    df['description_len'] = df['description'].fillna("").apply(lambda x: len(x.split()))

    # length of title
    df['title_len'] = df['title'].fillna("").apply(lambda x: len(x.split()))

    # param_combined and its length
    df['param_combined'] = df.apply(
        lambda row: ' '.join([str(row['param_1']), str(row['param_2']), str(row['param_3'])]), axis=1)
    df['param_combined_len'] = df['param_combined'].fillna("").apply(lambda x: len(x.split()))

    # charater len of text columns
    df['description_char'] = df['description'].fillna("").apply(len)
    df['title_char'] = df['title'].fillna("").apply(len)
    df['param_char'] = df['param_combined'].fillna("").apply(len)
    for col in ["description", "title"]:
        df[col+'_num_words'] = df[col].fillna("").apply(lambda comment: len(comment.split()))
        df[col+'_num_unique_words']=df[col].fillna("").apply(lambda comment:len(set(w for w in comment.split())))
        df[col+'_words_vs_unique']=df[col+'_num_unique_words']/df[col+'_num_words']*100

    return df

def get_geo_feature(df):
    df['city_region'] = df["city"] + ' ' + df["region"]
    city_region = pd.DataFrame()
    city_region['city_region'] = df['city_region'].drop_duplicates().reset_index(drop=True)
    city_region['latitude'] = np.nan
    city_region['longitude'] = np.nan

    from geopy import geocoders
    g = geocoders.GoogleV3()

    def get_geocode(row):
        try:
            geocode = g.geocode(row['city_region'],timeout=60,language='en')
            row['latitude'] = geocode.latitude
            row['longitude'] = geocode.longitude
        except:
            row['latitude'] = np.nan
            row['longitude'] = np.nan
        return row

    city_region = city_region.apply(get_geocode,axis=1)

    df = df.merge(city_region, on='city_region', how='left')
    df.drop(['city_region'],axis=1,inplace=True)
    return df

def tfidf_feat(train,test,features,n_components,analyzer,alg='pca',sparse=False,gram=3):

    print(features,n_components)

    pool = mlp.Pool(len(features))
    results = []
    for feat,k,anal in zip(features,n_components,analyzer):
        result = pool.apply_async(tfidf_worker,
                                  args=(feat,k,train[feat],test[feat],alg,sparse,anal,gram))
        results.append((feat,k,result))
        # tfidf_worker(feat,k,train[feat],test[feat],alg)
    pool.close()
    pool.join()

    for feat,k,result in tqdm(results):
        result.get()

def agg_time_feat(train,test):
    used_cols = ['item_id', 'user_id']

    train_sub = train[used_cols]
    train_active = pd.read_csv('./dataset/train_active.csv', usecols=used_cols)
    test_sub = test[used_cols]
    test_active = pd.read_csv('./dataset/test_active.csv', usecols=used_cols)

    train_periods = pd.read_csv('./dataset/periods_train.csv', parse_dates=['date_from', 'date_to'])
    test_periods = pd.read_csv('./dataset/periods_test.csv', parse_dates=['date_from', 'date_to'])

    all_samples = pd.concat([
        train_sub,
        train_active,
        test_sub,
        test_active
    ]).reset_index(drop=True)
    all_samples.drop_duplicates(['item_id'], inplace=True)

    all_periods = pd.concat([train_periods,test_periods])
    all_periods['days_up'] = all_periods['date_to'].dt.dayofyear - all_periods['date_from'].dt.dayofyear

    gp = all_periods.groupby(['item_id'])[['days_up']]

    gp_df = pd.DataFrame()
    gp_df['days_up_sum'] = gp.sum()['days_up']
    gp_df['times_put_up'] = gp.count()['days_up']
    gp_df.reset_index(inplace=True)
    gp_df.rename(index=str, columns={'index': 'item_id'})

    all_periods.drop_duplicates(['item_id'], inplace=True)
    all_periods = all_periods.merge(gp_df, on='item_id', how='left')

    all_periods = all_periods.merge(all_samples, on='item_id', how='left')

    gp = all_periods.groupby(['user_id'])[['days_up_sum', 'times_put_up']].mean().reset_index() \
        .rename(index=str, columns={
        'days_up_sum': 'avg_days_up_user',
        'times_put_up': 'avg_times_up_user'
    })

    n_user_items = all_samples.groupby(['user_id'])[['item_id']].count().reset_index() \
                        .rename(index=str, columns={'item_id': 'n_user_items'})
    gp = gp.merge(n_user_items, on='user_id', how='outer')

    train = train.merge(gp, on='user_id', how='left')
    test = test.merge(gp, on='user_id', how='left')

    return train,test

def get_img_info(df,datatype):
    from process_worker import img_info_worker
    if datatype=='tr':
        path = './dataset/train_img/'
    elif datatype=='te':
        path = './dataset/test_img/'
    else:
        raise RuntimeError("don't have this data")

    img_files = df['image'].values
    num_cpu = mlp.cpu_count()
    pool = mlp.Pool(num_cpu)
    num_task = 1+len(img_files)//num_cpu
    results = []

    for i in range(num_cpu):
        result = pool.apply_async(img_info_worker,
                        args=(img_files[i*num_task:(i+1)*num_task],path))
        results.append(result)
    pool.close()
    pool.join()

    img_feat = []
    for result in tqdm(results):
        img_feat += result.get()

    assert len(df) == img_feat
    columns = ['light_percent', 'dark_percent','average_color',
               'blurrness_score','dominant_color','average_pixel_width']
    img_feat = pd.DataFrame(img_feat,columns=columns)
    img_feat.to_csv('./dataset/'+datatype+'_info.csv',index=False)

def get_img_confi(df, datatype):
    from process_worker import img_confi_worker
    if datatype == 'tr':
        path = './dataset/train_img/'
    elif datatype == 'te':
        path = './dataset/test_img/'
    else:
        raise RuntimeError("don't have this data")

    img_files = df['image'].values
    num_cpu = mlp.cpu_count()
    pool = mlp.Pool(num_cpu)
    num_task = 1 + len(img_files) // num_cpu

    img_feat = pd.DataFrame()
    for model in ['resnet50','xception','inception_v3']:
        results = []
        for i in range(num_cpu):
            result = pool.apply_async(img_confi_worker,
                                      args=(img_files[i*num_task:(i + 1)*num_task],model,path))
            results.append(result)
        pool.close()
        pool.join()

        feat = []
        for result in tqdm(results):
            feat += result.get()

        assert len(df) == img_feat
        columns = [model+str(i) for i in range(3)]
        feat = pd.DataFrame(feat, columns=columns)
        img_feat = pd.concat([img_feat,feat],axis=1)

    img_feat.to_csv('./dataset/' + datatype + '_confi.csv', index=False)

def comb_feat():
    #  共现概率特征
    co_features = []
    co_prob_feat = [
        "category_name",
        "city",
        "param_1",
        "param_2",
        "param_3",
        "user_id",
        "weekday",
    ]
    for i in range(1, 4):
        co_features += list(combinations(co_prob_feat, i))

    # 条件概率特征
    from itertools import product
    condi_features = []

    area = ["city"]
    user_type = ["user_type"]
    user_id = ['user_id']
    date = ["activation_date", "weekday"]
    item = ["param_1", "category_name"]

    comb_feat = [area, user_type, date, item, user_id]
    for i in range(1, 2):
        for j in range(1, 3):
            for condi_varis in combinations(comb_feat, i):
                X = list(product(*condi_varis))
                res = [i for i in comb_feat if i not in condi_varis]
                for res_varis in combinations(res, j):
                    Y = list(product(*res_varis))
                    condi_features += list(product(X, Y))

    return co_features,condi_features

def prob_feature(train,test):

    def run_task(train,test,usecols,features,is_co):
        num_cpu = mlp.cpu_count()//2
        pool = mlp.Pool(num_cpu)
        num_task = 1 + len(features) // num_cpu
        results = []

        for i in range(num_cpu):
            result = pool.apply_async(prob_feature_worker,
                                      args=(usecols,features[i * num_task:(i + 1) * num_task],is_co))
            results.append(result)
        pool.close()
        pool.join()

        for result in tqdm(results):
            train_feat,test_feat = result.get()
            train = pd.concat([train,train_feat],axis=1)
            test = pd.concat([test,test_feat],axis=1)

        return train,test

    l_tr = len(train)
    l_te = len(test)
    usecols = [
        "item_id",
        "user_id",
        "region",
        "city",
        "parent_category_name",
        "category_name",
        "param_1",
        "param_2",
        "param_3",
        "activation_date",
        "user_type",
        "weekday"
    ]

    co_features,condi_features = comb_feat()
    train,test = run_task(train,test,usecols,co_features,True)

    train, test = run_task(train,test,usecols,condi_features,False)

    assert l_tr == len(train)
    assert l_te == len(test)

    return train,test

def user_item_matrix(train,test,dims):

    all_samples = pd.concat([
        train,
        test
    ]).reset_index(drop=True)

    def feat_encode(all_samples):

        from sklearn.preprocessing import LabelEncoder
        le = LabelEncoder()
        all_samples['city_id'] = le.fit_transform(all_samples['city'].values)
        le = LabelEncoder()
        all_samples['user_type_id'] = le.fit_transform(all_samples['user_type'].values)
        le = LabelEncoder()
        all_samples['category_name_id'] = le.fit_transform(all_samples['category_name'].values)

        all_samples['image_top_1_id'] = all_samples['image_top_1']
        all_samples['image_top_1_id'].fillna(all_samples['image_top_1'].max()+1, inplace=True)
        return all_samples

    all_samples = feat_encode(all_samples)

    i = len(all_samples['city_id'].unique())
    j = len(all_samples['user_type_id'].unique())
    assert all_samples['city_id'].max() == i-1
    assert all_samples['user_type_id'].max() == j-1
    num_user = i*j
    all_samples['u_id'] = i*all_samples['user_type_id'] + all_samples['city_id']
    all_samples['u_id'] = all_samples['u_id'].astype(int)

    i = int(all_samples['image_top_1_id'].max()+1)
    j = len(all_samples['category_name_id'].unique())
    all_samples['i_id'] = i * all_samples['category_name_id'] + all_samples['image_top_1_id']
    all_samples['i_id'] = all_samples['i_id'].astype(int)
    num_item = i*j

    ui_graph = all_samples[['u_id','i_id']].values

    u_i_matrix = np.zeros((num_user,num_item))
    print(u_i_matrix.shape)
    for i,j in ui_graph:
        u_i_matrix[i,j] +=1

    from scipy.sparse.linalg import svds
    [u,sigma,v] = svds(u_i_matrix,dims)
    v = v.T

    for i in range(dims):
        all_samples['user_vec_'+str(i)] = 0
        all_samples['item_vec_'+str(i)] = 0
    user_cols = ['user_vec_'+str(i) for i in range(dims)]
    item_cols = ['item_vec_'+str(i) for i in range(dims)]

    print('set vec')
    def set_vec(row):
        row[user_cols] = u[row['u_id']]
        row[item_cols] = v[row['i_id']]
        return row
    usecol = ['city', 'user_type', 'category_name', 'image_top_1']
    all_samples.drop_duplicates(usecol, inplace=True)
    all_samples = all_samples.apply(set_vec,axis=1)

    print('merge')
    all_samples = all_samples[usecol+user_cols+item_cols]
    train = train.merge(all_samples, on=usecol, how='left')
    test = test.merge(all_samples, on=usecol, how='left')

    for data,name in zip([train,test],['tr','te']):
        u = data[user_cols].values
        i = data[item_cols].values
        np.save('./dataset/'+name+'_u_id.csv',u)
        np.save('./dataset/'+name+'_i_id.csv',i)

def get_stat_features():

    cate_cols = [
        "city",
        "category_name",
        "param_1",
        "activation_date",
        "user_type",
        "user_id",
        "weekday",
    ]

    conti_cols = [
        "price",
        'item_seq_number',
    ]

    features = []
    for i in range(1, 3):
        for comb_feat in combinations(cate_cols, i):
            for conti_feat in conti_cols:
                features.append([conti_feat,list(comb_feat)])
    usecols = cate_cols + conti_cols + ['item_id']
    return usecols,features

def conti_stat_feature(train,test):

    l_train = len(train)
    l_test = len(test)

    usecols,features = get_stat_features()

    num_cpu = mlp.cpu_count()//2
    pool = mlp.Pool(num_cpu)
    num_task = 1+len(features)//num_cpu
    results = []

    for i in range(num_cpu):
        result = pool.apply_async(conti_stat_feature_worker,
                        args=(usecols,features[i*num_task:(i+1)*num_task]))
        results.append(result)
    pool.close()
    pool.join()

    for result in tqdm(results):
        train_feat,test_feat = result.get()
        train = pd.concat([train,train_feat],axis=1)
        test = pd.concat([test,test_feat],axis=1)

    assert len(train) == l_train
    assert len(test) == l_test

    return train,test

# def num_to_bin_feature(train,test):
#     samples = train[['price', 'item_seq_number']].append(test[['price', 'item_seq_number']]).reset_index(drop=True)
#
#     for num_bin,col in [()]
#     samples = num2bin(samples, ['price', 'item_seq_number'], 500)
#     train['price_bin'] = samples.loc[:len(train) - 1, 'price_bin']
#     test['price_bin'] = samples.loc[len(train):, 'price_bin']
#     train["item_seq_number_bin"] = samples.loc[:len(train) - 1, "item_seq_number_bin"]
#     test["item_seq_number_bin"] = samples.loc[len(train):, "item_seq_number_bin"]
#
#     return train, test

def target_encode(train,test):
    cate_cols = [
        "city",
        "category_name",
        "param_1",
        "activation_date",
        "user_type",
        "weekday",
        "image_top_1",
        # "price_bin",
        # "item_seq_number_bin"
    ]

    for col in tqdm(cate_cols):
        feat = condition_stat(train[[col,'deal_probability']],'deal_probability',col,col+'_TE_mean','mean')
        feat = feat[[col,col+'_TE_mean']].drop_duplicates([col]).reset_index(drop=True)
        train = train.merge(feat,how='left',on=[col])
        test = test.merge(feat, how='left', on=[col])

        feat = condition_stat(train[[col, 'deal_probability']], 'deal_probability', col, col +'_TE_std', 'std')
        feat = feat[[col,col+'_TE_std']].drop_duplicates([col]).reset_index(drop=True)
        train = train.merge(feat, how='left', on=[col])
        test = test.merge(feat, how='left', on=[col])
    return train,test

def oof_feature(train,test,alpha):
    NFOLDS = 5
    SEED = 42

    class SklearnWrapper(object):
        def __init__(self, clf, seed=0, params=None, seed_bool=True):
            if (seed_bool == True):
                params['random_state'] = seed
            self.clf = clf(**params)

        def train(self, x_train, y_train):
            self.clf.fit(x_train, y_train)

        def predict(self, x):
            return self.clf.predict(x)

    def get_oof(clf, x_train, y, x_test):

        from sklearn.cross_validation import KFold

        oof_train = np.zeros((len(y),))
        oof_test = np.zeros((len(test),))
        oof_test_skf = np.empty((NFOLDS,len(test)))

        kf = KFold(len(y), n_folds=NFOLDS, shuffle=True, random_state=SEED)
        for i, (train_index, test_index) in enumerate(kf):
            print('\nFold {}'.format(i))
            x_tr = x_train[train_index]
            y_tr = y[train_index]
            x_te = x_train[test_index]

            clf.train(x_tr, y_tr)

            oof_train[test_index] = clf.predict(x_te)
            oof_test_skf[i, :] = clf.predict(x_test)

        oof_test[:] = oof_test_skf.mean(axis=0)
        return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)

    print('load data')
    desc_matrix = np.load('./dataset/description.npy')
    titl_matrix = np.load('./dataset/title.npy')
    ready_df = np.concatenate((desc_matrix,titl_matrix),axis=1)

    print('train')
    ridge_params = {'alpha':alpha, 'fit_intercept': True, 'normalize': False, 'copy_X': True,
                    'max_iter': None, 'tol': 0.0001, 'solver': 'auto', 'random_state': SEED}
    y = train['deal_probability'].values
    ridge = SklearnWrapper(clf=Ridge, seed=SEED, params=ridge_params)
    ridge_oof_train, ridge_oof_test = get_oof(ridge, ready_df[:len(train)], y, ready_df[len(train):])

    from sklearn.metrics import mean_squared_error
    from math import sqrt
    rms = sqrt(mean_squared_error(y, ridge_oof_train))
    print('Ridge OOF RMSE: {}'.format(rms))

    train['ridge_preds'] = ridge_oof_train
    test['ridge_preds'] = ridge_oof_test

    return train,test

def generator(imgs,batchsize):
    from keras.preprocessing import image
    from keras.applications.vgg16 import preprocess_input

    iters = 1+len(imgs)//batchsize
    for i in tqdm(range(iters)):
        data = []
        for f in imgs[i*(batchsize):(i+1)*batchsize]:
            im = image.img_to_array(image.load_img(f,target_size=(256,256)))
            data.append(preprocess_input(im))

        yield {'input_1':np.array(data)}

def img_feat(train,test):
    import tensorflow as tf
    import keras.backend.tensorflow_backend as KTF
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True  # 不全部占满显存, 按需分配
    session = tf.Session(config=config)
    KTF.set_session(session)
    from keras.applications.vgg16 import VGG16
    from keras.models import Model
    from keras.layers import GlobalAveragePooling2D, GlobalMaxPool2D, concatenate
    from sklearn.decomposition import PCA

    out = VGG16(weights='imagenet',include_top=False)
    x = GlobalAveragePooling2D()(out.output)
    y = GlobalMaxPool2D()(out.output)
    z = concatenate([x,y])
    model = Model(inputs=out.input,outputs=z)

    train['image'].fillna('unk',inplace=True)
    train['image'] = './dataset/train_img/'+train['image']+'.jpg'
    test['image'].fillna('unk', inplace=True)
    test['image'] = './dataset/test_img/'+test['image']+'.jpg'
    train = train.append(test)
    imgs = train['image'].values
    del train,test
    print(len(imgs))
    print('predict')
    batchsize = 128
    steps = (len(imgs)+batchsize-1)//batchsize
    results = model.predict_generator(generator(imgs,batchsize=batchsize),steps=steps,verbose=1,use_multiprocessing=True)

    np.save('./dataset/img_feat.npy',results)

    pca = PCA(n_components=512)

    results = pca.fit_transform(results)
    np.save('./dataset/img_feat_demeni.npy', results)
    total_ratio = []
    ratio = 0
    for i in pca.explained_variance_ratio_:
        ratio += i
        total_ratio.append(ratio)
    print(total_ratio)

def get_word_vec(train,test,usecol):

    train = train[usecol]
    train_active = pd.read_csv('./dataset/train_active.csv', usecols=[usecol])
    test= test[usecol]
    test_active = pd.read_csv('./dataset/test_active.csv', usecols=[usecol])
    all_samples = pd.concat([
        train,
        train_active,
        test,
        test_active
    ]).fillna('unk').reset_index(drop=True)
    sentences = all_samples[usecol].values
    del train,train_active,test,test_active,all_samples
    gc.collect()

    from text_embedding import tokenize_word
    from gensim.models import Word2Vec
    from text_embedding import clean

    print('clean ')
    sentences = clean(sentences)
    num_seq = len(sentences)//4 + 1
    for i in range(4):
        sentences[i*num_seq:(i+1)*num_seq] = tokenize_word(sentences[i*num_seq:(i+1)*num_seq])

    print('train')
    model = Word2Vec(size=300,workers=32,iter=3,min_count=5)
    model.build_vocab(sentences)
    model.train(sentences,total_examples=model.corpus_count, epochs=3)  #
    del sentences
    print('save')
    model.save('./dataset/2'+usecol+'.model')
    del model
    print('success')
    gc.collect()

def pipeline():

    # train = pd.read_csv("./dataset/train.csv",parse_dates=["activation_date"])
    # test = pd.read_csv("./dataset/test.csv",parse_dates=["activation_date"])
    train = pd.read_csv("./dataset/train.csv",usecols=["description",'title'])
    test = pd.read_csv("./dataset/test.csv",usecols=["description",'title'])
    # train,test = baseFeature(train),baseFeature(test)
    # train,test = get_geo_feature(train),get_geo_feature(test)

    print('tfidf')
    # tfidf_feat(train, test, ['param_combined'],[1000,500],alg='ica')
    tfidf_feat(train,test,["description",'title'],[800,400],sparse=True,analyzer=['word','word'],gram=3)

    # get_word_vec(train,test,'description')
    # get_word_vec(train,test,'title')

    # img_feat(train,test)
    # train, test = agg_time_feat(train, test)

    # train,test = num_to_bin_feature(train,test)
    # train,test = target_encode(train,test)
    # get_img_info(train, 'tr')
    # get_img_info(test, 'te')
    # get_img_confi(train,'tr')
    # get_img_confi(test,'te')

    # train, test = oof_feature(train, test,20)

    # train, test = prob_feature(train, test)
    # train,test = conti_stat_feature(train,test)
    # user_item_matrix(train,test,100)

    # print('save')
    # train.to_csv("./dataset/train.csv",index=False)
    # test.to_csv("./dataset/test.csv",index=False)
    # print('save 2')
    # train.to_csv("./dataset/train.csv.gz", index=False,compression='gzip')
    # test.to_csv("./dataset/test.csv.gz", index=False,compression='gzip')

if __name__ == '__main__':
    pipeline()


from fabric.contrib.files import append, exists, sed
from fabric.api import env, local, run
import random

REPO_URL = 'https://github.com/tuvttran/TDD_Python.git'

def deploy():
    site_folder = f'/home/{env.user}/sites/{env.host}'
    source_folder = site_folder + '/source'
    _create_directory_structure_if_necessary(site_folder)
    _get_latest_source(source_folder)
    _update_settings(source_folder, env.host)
    _update_virtualenv(source_folder)
    _update_static_files(source_folder)
    _update_database(source_folder)

def _create_directory_structure_if_necessary(site_folder):
    for subfolder in ('database', 'static', 'virtualenv', 'source'):
        run(f'mkdir -p {site_folder}/{subfolder}')

def _get_latest_source(source_folder):
    if exists(source_folder + '/.git'):
        run(f'cd {source_folder} && git fetch')
    else:
        run(f'git clone {REPO_URL} {source_folder}')
    current_commit = local("git log -n 1 --format=%H", capture=True)
    run(f'cd {source_folder} && git reset --hard {current_commit}')

def _update_settings(source_folder, site_name):
    settings_path = source_folder + '/superlists/settings.py'
    sed(settings_path, "DEBUG = True", "DEBUG = False")
    sed(
        settings_path,
        'ALLOWED_HOSTS =.+$',
        f'ALLOWED_HOSTS = ["{site_name}"]'
    )
    secret_key_file = source_folder + '/superlists/secret_key.py'
    if not exists(secret_key_file):
        chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
        key = ''.join(random.SystemRandom().choice(chars) for _ in range(50))
        append(secret_key_file, f'SECRET_KEY = "{key}"')
    append(settings_path, '\nfrom .secret_key import SECRET_KEY')

def _update_virtualenv(source_folder):
    virtualenv_folder = source_folder + '/../virtualenv'
    if not exists(virtualenv_folder + '/bin/pip'):
        run(f'python3.6 -m venv {virtualenv_folder}')
    run(f'{virtualenv_folder}/bin/pip install -r {source_folder}/requirements.txt')

def _update_static_files(source_folder):
    run(
        f'cd {source_folder}'
        ' && ../virtualenv/bin/python manage.py collectstatic --noinput'
    )

def _update_database(source_folder):
    run(
        f'cd {source_folder}'
        ' && ../virtualenv/bin/python manage.py migrate --noinput'
    )


import random
from time import time

def randStr(len):
    string = ""
    for i in range(len):
        string += str(chr(ord('A') + random.randint(0, 25)))
    return string

def randNum(len):
    string = ""
    for i in range(len):
        string += str(random.randint(0, 9))
    return string

def breakString(str):
    return " ".join([c for c in str])

def whole(copy):
    for i in range(copy):
        numWord = random.randint(4, 7)
        numWord2 = random.randint(3, 6)
        numNum = random.randint(5, 8)
        numNum2 = random.randint(4, 6)

        yield "<s> %s - %s - %s - %s </s>\t<s> <WORD> - <WORD> - <NUM> - <NUM> </s>" % (
            breakString(randStr(numWord)), breakString(randStr(numWord2)), breakString(randNum(numNum)),
            breakString(randNum(numNum2)))

# If the group contains more than 3 different types, <WORD> will
# be used. Otherwise, a union will be used

copy = 1000

output = "data/whole.test"

f = open(output, "w")

random.seed(time())

for i in whole(copy):
    f.write("%s\n" % (i))

f.close()


import os
import re

# 打开并读取.gitmodules文件的内容
with open(".gitmodules", "r") as file:
    gitmodules_content = file.read()

def extract_submodules_info(gitmodules_content):
    # 使用正则表达式从.gitmodules内容中提取子模块的路径和URL
    pattern = r'\[submodule "(.*?)"\]\n\tpath = (.*?)\n\turl = (.*?)\n'
    return re.findall(pattern, gitmodules_content)

def display_submodules(submodules):
    # 显示所有可用的子模块
    print("Available submodules:")
    for idx, (name, path, url) in enumerate(submodules, 1):
        print(f"{idx}. {name} (Path: {path}, URL: {url})")

def clone_selected_submodules(submodules, selected_indices):
    # 根据用户选择的索引克隆选定的子模块
    for idx in selected_indices:
        name, path, url = submodules[idx-1]
        print(f"Cloning {name} into {path}...")
        os.system(f"git clone --depth=1 {url} {path}")

def main():
    try:
        print("Starting script...")
        # 从.gitmodules文件内容中提取子模块信息
        submodules = extract_submodules_info(gitmodules_content)
        # 显示所有子模块
        display_submodules(submodules)

        # 提示用户输入要克隆的子模块的编号
        selection = input("Enter the numbers of the submodules you want to clone (comma separated) or 'all' for all submodules: ")

        # 如果用户选择了"all"，则选择所有子模块，否则选择指定的子模块
        if selection == 'all':
            selected_indices = list(range(1, len(submodules) + 1))
        else:
            selected_indices = list(map(int, selection.split(',')))
        # 克隆选定的子模块
        clone_selected_submodules(submodules, selected_indices)

    except Exception as e:
        # 如果在执行过程中出现错误，打印错误信息
        print(f"An error occurred: {e}")

# 运行脚本
main()


# encoding: utf-8
"""
@author: Dianlei Zhang
@contact: dianlei.zhang@qq.com
@time: 2018/12/19 2:33 PM
"""

def singleNumber(nums):
    """
    :type nums: List[int]
    :rtype: int
    """
    result = 0
    for i in nums:
        result ^= i

    return result

if __name__ == '__main__':
    nums = [4, 1, 2, 1, 2, 5, 5]
    nums = [1]
    print(singleNumber(nums))