import os
from io import BytesIO
import subprocess  # nosec
from datetime import datetime, timezone

from sqlalchemy import and_

from flask import Blueprint, render_template, current_app, abort, request, flash, redirect, url_for, send_file
from flask_wtf import FlaskForm
from wtforms import SubmitField, HiddenField, StringField, SelectField, FileField
from wtforms.fields.html5 import DateTimeField, IntegerField, DecimalField
from wtforms.validators import DataRequired, NumberRange, Optional

from weconnect.errors import RetrievalError

from vwsfriend.model.trip import Trip
from vwsfriend.model.charging_session import ACDC, ChargingSession
from vwsfriend.model.refuel_session import RefuelSession
from vwsfriend.model.vehicle import Vehicle
from vwsfriend.model.settings import Settings, UnitOfLength, UnitOfTemperature
from vwsfriend.model.journey import Journey

from vwsfriend.util.location_util import locationFromLatLon, addCharger

bp = Blueprint('database', __name__, url_prefix='/database')

class SettingsEditForm(FlaskForm):
    id = HiddenField('id', validators=[Optional()])
    unit_of_length = HiddenField('unit_of_length', validators=[Optional()])
    unit_of_temperature = HiddenField('unit_of_temperature', validators=[Optional()])
    grafana_url = StringField('URL where the Grafana installation is reachable', validators=[DataRequired()])
    vwsfriend_url = StringField('URL where the VWsFriend installation is reachable', validators=[DataRequired()])
    locale = HiddenField('locale', validators=[Optional()])
    save = SubmitField('Save changes')

class TripEditForm(FlaskForm):
    id = HiddenField('id', validators=[Optional()])
    vehicle_vin = HiddenField('id', validators=[DataRequired()])
    startDate = DateTimeField('Start Date and Time (UTC)', validators=[DataRequired()])
    endDate = DateTimeField('End Date and Time (UTC)', validators=[DataRequired()])
    start_position_latitude = DecimalField('Start Position Latitude', places=10, validators=[Optional(), NumberRange(min=-90, max=90)])
    start_position_longitude = DecimalField('Start Position Longitude', places=10, validators=[Optional(), NumberRange(min=-180, max=180)])
    start_mileage_km = IntegerField('Start Mileage in km', validators=[Optional(), NumberRange(min=0)])
    destination_position_latitude = DecimalField('Destination Position Latitude', places=10, validators=[Optional(), NumberRange(min=-90, max=90)])
    destination_position_longitude = DecimalField('Destination Position Longitude', places=10, validators=[Optional(), NumberRange(min=-180, max=180)])
    end_mileage_km = IntegerField('End Mileage in km', validators=[Optional(), NumberRange(min=0)])
    save = SubmitField('Save changes')
    add = SubmitField('Add Trip')
    delete = SubmitField('Delete Trip')

class ChargingSessionEditForm(FlaskForm):
    id = HiddenField('id', validators=[Optional()])
    vehicle_vin = HiddenField('id', validators=[DataRequired()])
    connected = DateTimeField('Plug Connection Date and Time (UTC)', validators=[Optional()])
    locked = DateTimeField('Plug Locked Date and Time (UTC)', validators=[Optional()])
    started = DateTimeField('Charging Start Date and Time (UTC)', validators=[DataRequired()])
    ended = DateTimeField('Charging End Date and Time (UTC)', validators=[Optional()])
    unlocked = DateTimeField('Plug Unlocked Date and Time (UTC)', validators=[Optional()])
    disconnected = DateTimeField('Plug Disconnect Date and Time (UTC)', validators=[Optional()])
    # maxChargeCurrentACSetting
    targetSOCSetting_pct = IntegerField('Target SoC Setting', validators=[Optional(), NumberRange(min=0, max=100)])
    maximumChargePower_kW = IntegerField('Maximum Power during session', validators=[Optional(), NumberRange(min=0)])
    acdc = SelectField("AC/DC", choices=ACDC.choices(), coerce=ACDC.coerce, validators=[DataRequired()])
    startSOC_pct = IntegerField('SoC at start', validators=[Optional(), NumberRange(min=0, max=100)])
    endSOC_pct = IntegerField('SoC at end', validators=[Optional(), NumberRange(min=0, max=100)])
    mileage_km = IntegerField('Mileage in km when charging', validators=[Optional(), NumberRange(min=0)])
    position_latitude = DecimalField('Position Latitude', places=10, validators=[Optional(), NumberRange(min=-90, max=90)])
    position_longitude = DecimalField('Position Longitude', places=10, validators=[Optional(), NumberRange(min=-180, max=180)])
    charger_id = SelectField("Charger", validators=[Optional()])
    realCharged_kWh = DecimalField('Real kWh charged', places=4, validators=[Optional(), NumberRange(min=0, max=1000)])
    realCost_ct = IntegerField('Real cost in cents', validators=[Optional(), NumberRange(min=0)])

    save = SubmitField('Save changes')
    add = SubmitField('Add Session')
    delete = SubmitField('Delete Session')

class RefuelSessionEditForm(FlaskForm):
    id = HiddenField('id', validators=[Optional()])
    vehicle_vin = HiddenField('id', validators=[DataRequired()])
    date = DateTimeField('Date and Time (UTC)', validators=[Optional()])
    startSOC_pct = IntegerField('SoC at start', validators=[Optional(), NumberRange(min=0, max=100)])
    endSOC_pct = IntegerField('SoC at end', validators=[Optional(), NumberRange(min=0, max=100)])
    mileage_km = IntegerField('Mileage in km when charging', validators=[Optional(), NumberRange(min=0)])
    position_latitude = DecimalField('Position Latitude', places=10, validators=[Optional(), NumberRange(min=-90, max=90)])
    position_longitude = DecimalField('Position Longitude', places=10, validators=[Optional(), NumberRange(min=-180, max=180)])
    realRefueled_l = DecimalField('Real l refueled', places=4, validators=[Optional(), NumberRange(min=0, max=1000)])
    realCost_ct = IntegerField('Real cost in cents', validators=[Optional(), NumberRange(min=0)])

    save = SubmitField('Save changes')
    add = SubmitField('Add Session')
    delete = SubmitField('Delete Session')

class JourneyEditForm(FlaskForm):
    id = HiddenField('id', validators=[Optional()])
    vehicle_vin = HiddenField('id', validators=[DataRequired()])
    start = DateTimeField('Start date and Time (UTC)', validators=[DataRequired()])
    end = DateTimeField('End date and Time (UTC)', validators=[DataRequired()])
    title = StringField('Title', validators=[DataRequired()])
    description = StringField('Description', validators=[DataRequired()])

    save = SubmitField('Save changes')
    add = SubmitField('Add journey')
    delete = SubmitField('Delete journey')

class BackupRestoreForm(FlaskForm):
    file = FileField('Restore file', validators=[Optional()])

    restore = SubmitField('restore')
    backup = SubmitField('backup')

@bp.route('/settings/edit', methods=['GET', 'POST'])
def settingsEdit():
    settings = current_app.db.session.query(Settings).first()
    if settings is None:
        requestHost = request.headers.get('Host')
        grafanaHost = requestHost.replace("4000", "3000")
        settings = Settings(grafana_url=f'http://{grafanaHost}', vwsfriend_url=f'http://{requestHost}')

    form = SettingsEditForm()

    if form.validate_on_submit():
        with current_app.db.session.begin_nested():
            settings.unit_of_length = UnitOfLength(form.unit_of_length.data)
            settings.unit_of_temperature = UnitOfTemperature(form.unit_of_temperature.data)
            settings.grafana_url = form.grafana_url.data
            settings.vwsfriend_url = form.vwsfriend_url.data
            settings.locale = form.locale.data
            current_app.db.session.merge(settings)
        current_app.db.session.commit()
        return redirect(url_for('status.vehicles'))

    form.id.data = settings.id
    form.unit_of_length.data = settings.unit_of_length.value
    form.unit_of_temperature.data = settings.unit_of_temperature.value
    form.grafana_url.data = settings.grafana_url
    form.vwsfriend_url.data = settings.vwsfriend_url
    form.locale.data = settings.locale
    return render_template('database/settings_edit.html', current_app=current_app, form=form)

@bp.route('/trips/edit', methods=['GET', 'POST'])  # noqa: C901
def tripEdit():  # noqa: C901
    id = None
    vin = None
    vehicle = None

    form = TripEditForm()

    if request.args is not None:
        id = request.args.get('id')
        vin = request.args.get('vin')
    if id is None and form.id.data is not None and form.id.data.isdigit():
        id = form.id.data
    trip = None

    if id is not None:
        trip = current_app.db.session.query(Trip).filter(Trip.id == id).first()

        if trip is None:
            abort(404, f"Trip with id {id} doesn't exist.")
    elif vin is None:
        flash(message='You need to provide a vin to add a trip', category='error')
    else:
        vehicle = current_app.db.session.query(Vehicle).filter(Vehicle.vin == vin).first()
        if vehicle is None:
            flash(message=f'Vehicle with VIN {vin} does not exist', category='error')

    if trip is not None and form.delete.data:
        flash(f'Successfully deleted trip {id}', category='info')
        with current_app.db.session.begin_nested():
            current_app.db.session.delete(trip)
        current_app.db.session.commit()
        return render_template('database/trip_edit.html', current_app=current_app, form=None)

    if trip is not None and form.save.data and form.validate_on_submit():
        with current_app.db.session.begin_nested():
            trip.startDate = form.startDate.data
            if trip.startDate is not None:
                trip.startDate = trip.startDate.replace(tzinfo=timezone.utc, microsecond=0)
            trip.endDate = form.endDate.data
            if trip.endDate is not None:
                trip.endDate = trip.endDate.replace(tzinfo=timezone.utc, microsecond=0)
            trip.start_position_latitude = form.start_position_latitude.data
            trip.start_position_longitude = form.start_position_longitude.data
            if trip.start_position_latitude is not None and trip.start_position_longitude is not None:
                trip.start_location = locationFromLatLon(current_app.db.session, trip.start_position_latitude, trip.start_position_longitude)
            trip.destination_position_latitude = form.destination_position_latitude.data
            trip.destination_position_longitude = form.destination_position_longitude.data
            if trip.destination_position_latitude is not None and trip.destination_position_longitude is not None:
                trip.destination_location = locationFromLatLon(current_app.db.session, trip.destination_position_latitude, trip.destination_position_longitude)
            trip.start_mileage_km = form.start_mileage_km.data
            trip.end_mileage_km = form.end_mileage_km.data
        current_app.db.session.commit()
        flash(message=f'Successfully updated trip {id}', category='info')

    elif trip is None and form.add.data and form.validate_on_submit():
        startDate = form.startDate.data
        if startDate is not None:
            startDate = startDate.replace(tzinfo=timezone.utc, microsecond=0)
        endDate = form.endDate.data
        if endDate is not None:
            endDate = endDate.replace(tzinfo=timezone.utc, microsecond=0)
        start_position_latitude = form.start_position_latitude.data
        start_position_longitude = form.start_position_longitude.data
        if start_position_latitude is not None and start_position_longitude is not None:
            start_location = locationFromLatLon(current_app.db.session, start_position_latitude, start_position_longitude)
        else:
            start_location = None
        destination_position_latitude = form.destination_position_latitude.data
        destination_position_longitude = form.destination_position_longitude.data
        if destination_position_latitude is not None and destination_position_longitude is not None:
            destination_location = locationFromLatLon(current_app.db.session, destination_position_latitude, destination_position_longitude)
        else:
            destination_location = None

        start_mileage_km = form.start_mileage_km.data
        end_mileage_km = form.end_mileage_km.data

        trip = Trip(vehicle, startDate, start_position_latitude, start_position_longitude, start_location, start_mileage_km)
        trip.endDate = endDate
        trip.destination_position_latitude = destination_position_latitude
        trip.destination_position_longitude = destination_position_longitude
        trip.destination_location = destination_location
        trip.end_mileage_km = end_mileage_km
        with current_app.db.session.begin_nested():
            current_app.db.session.add(trip)
        current_app.db.session.commit()
        current_app.db.session.refresh(trip)
        flash(message=f'Successfully added a new trip {trip.id}', category='info')

    if trip is not None:
        form.id.data = trip.id
        form.vehicle_vin.data = trip.vehicle_vin
        form.startDate.data = trip.startDate
        form.endDate.data = trip.endDate
        form.start_position_latitude.data = trip.start_position_latitude
        form.start_position_longitude.data = trip.start_position_longitude
        form.destination_position_latitude.data = trip.destination_position_latitude
        form.destination_position_longitude.data = trip.destination_position_longitude
        form.start_mileage_km.data = trip.start_mileage_km
        form.end_mileage_km.data = trip.end_mileage_km
    else:
        form.vehicle_vin.data = vehicle.vin
        form.startDate.data = datetime.utcnow()
        form.endDate.data = datetime.utcnow()

        lasttrip = current_app.db.session.query(Trip).filter(and_(Trip.vehicle == vehicle, Trip.startDate.isnot(None))).order_by(Trip.startDate.desc()).first()
        if lasttrip is not None and lasttrip.end_mileage_km is not None:
            form.start_mileage_km.data = lasttrip.end_mileage_km
            form.end_mileage_km.data = lasttrip.end_mileage_km

    return render_template('database/trip_edit.html', current_app=current_app, form=form)

@bp.route('/charging-session/edit', methods=['GET', 'POST'])  # noqa: C901
def chargingSessionEdit():  # noqa: C901
    id = None
    vin = None
    vehicle = None

    form = ChargingSessionEditForm()

    if request.args is not None:
        id = request.args.get('id')
        vin = request.args.get('vin')
    if id is None and form.id.data is not None and form.id.data.isdigit():
        id = form.id.data
    chargingSession = None

    if id is not None:
        chargingSession = current_app.db.session.query(ChargingSession).filter(ChargingSession.id == id).first()

        if chargingSession is None:
            abort(404, f"Charging session with id {id} doesn't exist.")

    elif vin is None:
        flash(message='You need to provide a vin to add a charging session', category='error')
    else:
        vehicle = current_app.db.session.query(Vehicle).filter(Vehicle.vin == vin).first()
        if vehicle is None:
            flash(message=f'Vehicle with VIN {vin} does not exist', category='error')

    try:
        if form.position_latitude.data is not None:
            latitude = form.position_latitude.data
        elif chargingSession is not None:
            latitude = chargingSession.position_latitude
        else:
            latitude = None

        if form.position_longitude.data is not None:
            longitude = form.position_longitude.data
        elif chargingSession is not None:
            longitude = chargingSession.position_longitude
        else:
            longitude = None

        if latitude is not None and longitude is not None:
            chargers = sorted(current_app.weConnect.getChargingStations(round(latitude, 4), round(longitude, 4), searchRadius=500).values(),
                              key=lambda station: station.distance.value)
            choices = [(None, 'unknown')]
            for charger in chargers:
                label = f'{charger.name.value} ({round(charger.distance.value)} m away)'
                choices.append((charger.id.value, label))
            form.charger_id.choices = choices
        else:
            form.charger_id.choices = [(None, 'You need to first save the position of the charging session')]
    except RetrievalError as e:
        flash(message=f'Cannot retrieve chargers: {e}', category='error')
        pass

    if chargingSession is not None and form.delete.data:
        flash(message=f'Successfully deleted charging session {id}', category='info')
        with current_app.db.session.begin_nested():
            current_app.db.session.delete(chargingSession)
        current_app.db.session.commit()
        return render_template('database/charging_session_edit.html', current_app=current_app, form=None)

    if chargingSession is not None and form.save.data and form.validate_on_submit():
        with current_app.db.session.begin_nested():
            chargingSession.connected = form.connected.data
            if chargingSession.connected is not None:
                chargingSession.connected = chargingSession.connected.replace(tzinfo=timezone.utc, microsecond=0)
            chargingSession.locked = form.locked.data
            if chargingSession.locked is not None:
                chargingSession.locked = chargingSession.locked.replace(tzinfo=timezone.utc, microsecond=0)
            chargingSession.started = form.started.data
            if chargingSession.started is not None:
                chargingSession.started = chargingSession.started.replace(tzinfo=timezone.utc, microsecond=0)
            chargingSession.ended = form.ended.data
            if chargingSession.ended is not None:
                chargingSession.ended = chargingSession.ended.replace(tzinfo=timezone.utc, microsecond=0)
            chargingSession.unlocked = form.unlocked.data
            if chargingSession.unlocked is not None:
                chargingSession.unlocked = chargingSession.unlocked.replace(tzinfo=timezone.utc, microsecond=0)
            chargingSession.disconnected = form.disconnected.data
            if chargingSession.disconnected is not None:
                chargingSession.disconnected = chargingSession.disconnected.replace(tzinfo=timezone.utc, microsecond=0)
            chargingSession.targetSOCSetting_pct = form.targetSOCSetting_pct.data
            chargingSession.maximumChargePower_kW = form.maximumChargePower_kW.data
            chargingSession.acdc = form.acdc.data
            chargingSession.startSOC_pct = form.startSOC_pct.data
            chargingSession.endSOC_pct = form.endSOC_pct.data
            chargingSession.mileage_km = form.mileage_km.data
            chargingSession.position_latitude = form.position_latitude.data
            chargingSession.position_longitude = form.position_longitude.data
            if chargingSession.position_latitude is not None and chargingSession.position_longitude is not None:
                chargingSession.location = locationFromLatLon(current_app.db.session, chargingSession.position_latitude, chargingSession.position_longitude)

            chargingSession.realCharged_kWh = form.realCharged_kWh.data
            chargingSession.realCost_ct = form.realCost_ct.data
            if form.charger_id.data is None or form.charger_id.data == 'None':
                chargingSession.charger = None
            else:
                chargerAdded = False
                for charger in chargers:
                    if charger.id.value == form.charger_id.data:
                        chargingSession.charger = addCharger(current_app.db.session, charger)
                        chargerAdded = True
                if not chargerAdded:
                    flash(message='Charger not valid anymore', category='error')

        current_app.db.session.commit()
        flash(message=f'Successfully updated charging session {id}', category='error')

    elif chargingSession is None and form.add.data and form.validate_on_submit():
        chargingSession = ChargingSession(vehicle)

        chargingSession.connected = form.connected.data
        chargingSession.locked = form.locked.data
        chargingSession.started = form.started.data
        chargingSession.ended = form.ended.data
        chargingSession.unlocked = form.unlocked.data
        chargingSession.disconnected = form.disconnected.data
        chargingSession.targetSOCSetting_pct = form.targetSOCSetting_pct.data
        chargingSession.maximumChargePower_kW = form.maximumChargePower_kW.data
        chargingSession.acdc = form.acdc.data
        chargingSession.startSOC_pct = form.startSOC_pct.data
        chargingSession.endSOC_pct = form.endSOC_pct.data
        chargingSession.mileage_km = form.mileage_km.data
        chargingSession.position_latitude = form.position_latitude.data
        chargingSession.position_longitude = form.position_longitude.data
        if chargingSession.position_latitude is not None and chargingSession.position_longitude is not None:
            chargingSession.location = locationFromLatLon(current_app.db.session, chargingSession.position_latitude, chargingSession.position_longitude)
        else:
            chargingSession.location = None
        if form.charger_id.data is None or form.charger_id.data == 'None':
            chargingSession.charger = None
        else:
            chargingSession.charger_id = form.charger_id.data
        chargingSession.realCharged_kWh = form.realCharged_kWh.data
        chargingSession.realCost_ct = form.realCost_ct.data

        with current_app.db.session.begin_nested():
            current_app.db.session.add(chargingSession)
        current_app.db.session.commit()
        current_app.db.session.refresh(chargingSession)
        flash(message=f'Successfully added a new charging session {chargingSession.id}', category='info')

    if chargingSession is not None:
        form.id.data = chargingSession.id
        form.vehicle_vin.data = chargingSession.vehicle_vin
        form.connected.data = chargingSession.connected
        form.locked.data = chargingSession.locked
        form.started.data = chargingSession.started
        form.ended.data = chargingSession.ended
        form.unlocked.data = chargingSession.unlocked
        form.disconnected.data = chargingSession.disconnected
        form.targetSOCSetting_pct.data = chargingSession.targetSOCSetting_pct
        form.maximumChargePower_kW.data = chargingSession.maximumChargePower_kW
        form.acdc.data = chargingSession.acdc
        form.startSOC_pct.data = chargingSession.startSOC_pct
        form.endSOC_pct.data = chargingSession.endSOC_pct
        form.mileage_km.data = chargingSession.mileage_km
        form.position_latitude.data = chargingSession.position_latitude
        form.position_longitude.data = chargingSession.position_longitude

        form.charger_id.data = chargingSession.charger_id
        form.realCharged_kWh.data = chargingSession.realCharged_kWh
        form.realCost_ct.data = chargingSession.realCost_ct
    else:
        form.vehicle_vin.data = vehicle.vin
        form.started.data = datetime.utcnow()
        form.ended.data = datetime.utcnow()

        lasttrip = current_app.db.session.query(Trip).filter(and_(Trip.vehicle == vehicle, Trip.startDate.isnot(None))).order_by(Trip.startDate.desc()).first()
        if lasttrip is not None and lasttrip.start_mileage_km is not None:
            form.mileage_km.data = lasttrip.start_mileage_km

        choices = [(None, 'You need to first save the location of the charging session')]
        form.charger_id.choices = choices

    return render_template('database/charging_session_edit.html', current_app=current_app, form=form)

@bp.route('/refuel-session/edit', methods=['GET', 'POST'])  # noqa: C901
def refuelSessionEdit():  # noqa: C901
    id = None
    vin = None
    vehicle = None

    form = RefuelSessionEditForm()

    if request.args is not None:
        id = request.args.get('id')
        vin = request.args.get('vin')
    if id is None and form.id.data is not None and form.id.data.isdigit():
        id = form.id.data
    refuelSession = None

    if id is not None:
        refuelSession = current_app.db.session.query(RefuelSession).filter(RefuelSession.id == id).first()

        if refuelSession is None:
            abort(404, f"Refuel session with id {id} doesn't exist.")

    elif vin is None:
        flash(message='You need to provide a vin to add a refuel session', category='error')
    else:
        vehicle = current_app.db.session.query(Vehicle).filter(Vehicle.vin == vin).first()
        if vehicle is None:
            flash(message=f'Vehicle with VIN {vin} does not exist', category='error')

    if refuelSession is not None and form.delete.data:
        flash(message=f'Successfully deleted refuel session {id}', category='info')
        with current_app.db.session.begin_nested():
            current_app.db.session.delete(refuelSession)
        current_app.db.session.commit()
        return render_template('database/refuel_session_edit.html', current_app=current_app, form=None)

    if refuelSession is not None and form.save.data and form.validate_on_submit():
        with current_app.db.session.begin_nested():
            refuelSession.date = form.date.data
            if refuelSession.date is not None:
                refuelSession.date = refuelSession.date.replace(tzinfo=timezone.utc, microsecond=0)
            refuelSession.startSOC_pct = form.startSOC_pct.data
            refuelSession.endSOC_pct = form.endSOC_pct.data
            refuelSession.mileage_km = form.mileage_km.data
            refuelSession.position_latitude = form.position_latitude.data
            refuelSession.position_longitude = form.position_longitude.data
            if refuelSession.position_latitude is not None and refuelSession.position_longitude is not None:
                refuelSession.location = locationFromLatLon(current_app.db.session, refuelSession.position_latitude, refuelSession.position_longitude)
            refuelSession.realRefueled_l = form.realRefueled_l.data
            refuelSession.realCost_ct = form.realCost_ct.data

        current_app.db.session.commit()
        flash(message=f'Successfully updated refuel session {id}', category='info')

    elif refuelSession is None and form.add.data and form.validate_on_submit():
        date = form.date.data
        if date is not None:
            date = date.replace(tzinfo=timezone.utc, microsecond=0)

        latitude = form.position_latitude.data
        longitude = form.position_longitude.data
        if latitude is not None and longitude is not None:
            location = locationFromLatLon(current_app.db.session, latitude, longitude)
        else:
            location = None
        refuelSession = RefuelSession(vehicle=vehicle, date=date, startSOC_pct=form.startSOC_pct.data, endSOC_pct=form.endSOC_pct.data,
                                      mileage_km=form.mileage_km.data, position_latitude=latitude, position_longitude=longitude, location=location)
        refuelSession.realRefueled_l = form.realRefueled_l.data
        refuelSession.realCost_ct = form.realCost_ct.data

        with current_app.db.session.begin_nested():
            current_app.db.session.add(refuelSession)
        current_app.db.session.commit()
        current_app.db.session.refresh(refuelSession)
        flash(message=f'Successfully added a new charging session {refuelSession.id}', category='info')

    if refuelSession is not None:
        form.id.data = refuelSession.id
        form.vehicle_vin.data = refuelSession.vehicle_vin
        form.date.data = refuelSession.date
        form.startSOC_pct.data = refuelSession.startSOC_pct
        form.endSOC_pct.data = refuelSession.endSOC_pct
        form.mileage_km.data = refuelSession.mileage_km
        form.position_latitude.data = refuelSession.position_latitude
        form.position_longitude.data = refuelSession.position_longitude
        form.realRefueled_l.data = refuelSession.realRefueled_l
        form.realCost_ct.data = refuelSession.realCost_ct
    else:
        form.vehicle_vin.data = vehicle.vin
        form.date.data = datetime.utcnow()

        lasttrip = current_app.db.session.query(Trip).filter(and_(Trip.vehicle == vehicle, Trip.startDate.isnot(None))).order_by(Trip.startDate.desc()).first()
        if lasttrip is not None and lasttrip.start_mileage_km is not None:
            form.mileage_km.data = lasttrip.start_mileage_km
        form.endSOC_pct.data = 100

    return render_template('database/refuel_session_edit.html', current_app=current_app, form=form)

@bp.route('/journey/edit', methods=['GET', 'POST'])  # noqa: C901
def journeyEdit():  # noqa: C901
    id = None
    vin = None
    vehicle = None

    form = JourneyEditForm()

    if request.args is not None:
        id = request.args.get('id')
        vin = request.args.get('vin')
    if id is None and form.id.data is not None and form.id.data.isdigit():
        id = form.id.data
    journey = None

    if id is not None:
        journey = current_app.db.session.query(Journey).filter(Journey.id == id).first()

        if journey is None:
            abort(404, f"Journey with id {id} doesn't exist.")

    elif vin is None:
        flash(message='You need to provide a vin to add a journey', category='error')
    else:
        vehicle = current_app.db.session.query(Vehicle).filter(Vehicle.vin == vin).first()
        if vehicle is None:
            flash(message=f'Vehicle with VIN {vin} does not exist', category='error')

    if journey is not None and form.delete.data:
        flash(message=f'Successfully deleted journey {id}', category='info')
        with current_app.db.session.begin_nested():
            current_app.db.session.delete(journey)
        current_app.db.session.commit()
        return render_template('database/journey_edit.html', current_app=current_app, form=None)

    if journey is not None and form.save.data and form.validate_on_submit():
        with current_app.db.session.begin_nested():
            journey.start = form.start.data
            journey.end = form.end.data
            journey.title = form.title.data
            journey.description = form.description.data

        current_app.db.session.commit()
        flash(message=f'Successfully updated journey {id}', category='info')

    elif journey is None and form.add.data and form.validate_on_submit():
        start = form.start.data
        if start is not None:
            start = start.replace(tzinfo=timezone.utc, microsecond=0)
        end = form.end.data
        if end is not None:
            end = end.replace(tzinfo=timezone.utc, microsecond=0)

        journey = Journey(vehicle=vehicle, start=start, end=end, title=form.title.data)
        journey.description = form.description.data

        with current_app.db.session.begin_nested():
            current_app.db.session.add(journey)
        current_app.db.session.commit()
        current_app.db.session.refresh(journey)
        flash(message=f'Successfully added a new journey {journey.id}', category='info')

    if journey is not None:
        form.id.data = journey.id
        form.vehicle_vin.data = journey.vehicle_vin
        form.start.data = journey.start
        form.end.data = journey.end
        form.title.data = journey.title
        form.description.data = journey.description
    else:
        form.vehicle_vin.data = vehicle.vin
        if request.args is not None:
            tripstart = request.args.get('tripstart')
            if tripstart is not None:
                form.start.data = datetime.utcfromtimestamp(int(tripstart) / 1000)
            tripend = request.args.get('tripend')
            if tripend is not None:
                form.end.data = datetime.utcfromtimestamp(int(tripend) / 1000)

    return render_template('database/journey_edit.html', current_app=current_app, form=form)

@bp.route('/backup', methods=['GET', 'POST'])  # noqa: C901
def backup():  # noqa: C901
    form = BackupRestoreForm()

    if form.restore.data and form.validate_on_submit():
        if not form.file.data:
            flash('No file part')
        else:
            file = form.file.data
            # If the user does not select a file, the browser submits an
            # empty file without a filename.
            if file.filename == '':
                flash('No selected file')
            elif file and file.filename.endswith('.vwsfrienddbbackup'):
                if current_app.configDir is None or not os.path.isdir(current_app.configDir):
                    flash('Config directory is not configured or does not exist')
                else:
                    if not os.path.isdir(current_app.configDir + '/provisioning'):
                        os.makedirs(current_app.configDir + '/provisioning')
                    form.file.data.save(current_app.configDir + '/provisioning/database.vwsfrienddbbackup')
                    flash('backup was uploaded, now restarting to apply the backup')
                    return redirect(url_for('restart'))
            else:
                flash('File needs to be a .vwsfrienddbbackup')
    elif form.backup.data:
        try:
            dburl = current_app.config['SQLALCHEMY_DATABASE_URI']
            process = subprocess.run(['pg_dump', '--compress', '9', '--format', 'c', dburl], stdout=subprocess.PIPE, stderr=subprocess.PIPE)  # nosec

            if process.returncode != 0:
                return abort(500, f"pg_dump returned {process.returncode}: {process.stderr.decode('ascii')}")
            return send_file(BytesIO(process.stdout), mimetype='application/gzip', as_attachment=True,
                             download_name=f'{datetime.now().strftime("%Y%m%dT%H%M%S")}.vwsfrienddbbackup')
        except FileNotFoundError as e:
            return abort(500, f"pg_dump was not found, you have to install it in order to be able to make database backups: {e}")

    return render_template('database/backup.html', current_app=current_app, form=form)


from pytest import raises
from . import (
    HttpJsonResponse,
    check_github_api_response_for_errors,
    sut_module_target,
)

exp_owner = "swellaby"
exp_repository = "letra"
raise_github_repository_not_found_error_mock_target = (
    f"{sut_module_target}.raise_github_repository_not_found_error"
)
raise_github_unauthorized_error_mock_target = (
    f"{sut_module_target}.raise_github_unauthorized_error"
)

def call_sut_function(resp):
    return check_github_api_response_for_errors(
        response=resp,
        owner=exp_owner,
        repository=exp_repository,
        request_headers={},
    )

def test_returns_on_200_status_code():
    resp = HttpJsonResponse(status=200, headers={}, data="")
    assert call_sut_function(resp) is None

def test_returns_on_201_status_code():
    resp = HttpJsonResponse(status=201, headers={}, data="")
    assert call_sut_function(resp) is None

def test_returns_on_202_status_code():
    resp = HttpJsonResponse(status=202, headers={}, data="")
    assert call_sut_function(resp) is None

def test_returns_on_203_status_code():
    resp = HttpJsonResponse(status=203, headers={}, data="")
    assert call_sut_function(resp) is None

def test_returns_on_204_status_code():
    resp = HttpJsonResponse(status=204, headers={}, data="")
    assert call_sut_function(resp) is None

def test_raises_correct_err_on_404_status_code(monkeypatch):
    resp = HttpJsonResponse(status=404, headers={}, data="")
    exp_err = "repository not found"
    exp_headers = {"Authorization": "token 9999999"}
    act_owner = ""
    act_repository = ""
    act_req_headers = {}

    def mock_not_found(owner: str, repository: str, request_headers: dict):
        nonlocal act_owner, act_repository, act_req_headers
        act_owner = owner
        act_repository = repository
        act_req_headers = request_headers
        raise ValueError(exp_err)

    monkeypatch.setattr(
        f"{sut_module_target}.raise_github_repository_not_found_error",
        mock_not_found,
    )

    with raises(ValueError) as err:
        check_github_api_response_for_errors(
            response=resp,
            owner=exp_owner,
            repository=exp_repository,
            request_headers=exp_headers,
        )

    assert str(err.value) == exp_err
    assert act_owner == exp_owner
    assert act_repository == exp_repository
    assert act_req_headers == exp_headers

def test_raises_correct_err_on_401_status_code():
    exp_err = (
        f"Invalid authentication. Unable "
        f"to access GitHub repository: {exp_owner}/{exp_repository}"
    )
    resp = HttpJsonResponse(status=401, headers={}, data="")

    with raises(ValueError) as err:
        check_github_api_response_for_errors(
            response=resp,
            owner=exp_owner,
            repository=exp_repository,
            request_headers={},
        )

    assert str(err.value) == exp_err

def test_raises_correct_err_on_403_status_code(monkeypatch):
    resp = HttpJsonResponse(status=403, headers={}, data="")
    exp_err = "not authorized"
    exp_headers = {"Authorization": "token 123987"}
    act_response = {}
    act_owner = ""
    act_repository = ""
    act_req_headers = {}

    def mock_not_authorized(response, owner, repository, request_headers):
        nonlocal act_response, act_owner, act_repository, act_req_headers
        act_response = response
        act_owner = owner
        act_repository = repository
        act_req_headers = request_headers
        raise IOError(exp_err)

    mock_target = f"{sut_module_target}.raise_github_unauthorized_error"
    monkeypatch.setattr(mock_target, mock_not_authorized)

    with raises(IOError) as err:
        check_github_api_response_for_errors(
            response=resp,
            owner=exp_owner,
            repository=exp_repository,
            request_headers=exp_headers,
        )

    assert str(err.value) == exp_err
    assert act_response == resp
    assert act_owner == exp_owner
    assert act_repository == exp_repository
    assert act_req_headers == exp_headers

def test_raises_correct_err_on_unknown_status_code():
    status_code = 500
    exp_err = (
        "Unexpected HTTP error encountered from GitHub API. "
        f"HTTP response status code: {status_code}"
    )
    resp = HttpJsonResponse(status=status_code, headers={}, data="")

    with raises(IOError) as err:
        check_github_api_response_for_errors(
            response=resp,
            owner=exp_owner,
            repository=exp_repository,
            request_headers={},
        )

    assert str(err.value) == exp_err


# uncompyle6 version 3.7.4
# Python bytecode 2.4 (62061)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.linux-i686/egg/itcc/molecule/txyz.py
# Compiled at: 2008-04-20 13:19:45
import numpy

def get_coords(ifile, idxs=None):
    line = ifile.readlines()
    if idxs is None:
        idxs = range(int(line.split()[0]))
    max_idx = max(idxs) + 1
    lines = []
    for i in range(max_idx):
        lines.append(ifile.readline())

    return numpy.array([ [ float(x) for x in lines[i].split()[2:5] ] for i in idxs ])


from model import MixMatchVAE
import torch
from scipy.interpolate import interp1d
import numpy as np
import torchvision

def sample():
    n_sample = 20
    x, y = next(iter(testloader))
    result = torch.zeros(x.shape[0] * n_sample, 1, 28, 28).to(device)
    C = torch.zeros(x.shape[0] * n_sample, 1, 28, 28).to(device)
    z_val = [z for z in np.arange(-2, 2, 0.1)]
    with torch.no_grad():
        cnt = 0
        for i in range(x.shape[0]):
            reshaped_image = x[i:i+1].view(1, -1)
            data, condition = reshaped_image[:, :500], \
                              reshaped_image[:, 500:]
            data, condition = data.to(device), condition.to(device)
            result[cnt] = x[i]
            cnt += 1
            C_image = torch.zeros(reshaped_image.shape).to(device)
            C_image[:, 500:] = condition
            C_image = C_image.view(x[i:i + 1].shape)
            result[cnt] = C_image
            cnt += 1
            for j in range(2, n_sample):
                z = torch.ones(1, 32).to(device)
                z *= z_val[j]
                reconstructed = model.sample(condition, 64)
                rec_image = torch.zeros(reshaped_image.shape).to(device)

                rec_image[:, :500] = reconstructed
                rec_image[:, 500:] = condition

                rec_image = rec_image.view(x[i:i+1].shape)
                result[cnt] = rec_image
                cnt += 1
    torchvision.utils.save_image(result, nrow=20, fp="results/sampled.png")

if __name__ == "__main__":
    device = "cuda"

    class MixMatchArgs(object):
        def __init__(self):
            self.input_dim = 500  # (28 * 28) // 2
            self.output_dim = 500  # (28 * 28) // 2
            self.condition_dim = (28 * 28) - 500
            self.hidden_dim = 128
            self.latent_dim = 32
            self.device = device

    model = MixMatchVAE(MixMatchArgs()).to(device)

    transform = torchvision.transforms.Compose([
        torchvision.transforms.ToTensor(),
    ])

    testset = torchvision.datasets.MNIST(root='./data', train=False,
                                         download=True, transform=transform)
    testloader = torch.utils.data.DataLoader(testset, batch_size=20,
                                             shuffle=False, num_workers=0)

    model.load_state_dict(torch.load("checkpoint/mix_match_48.pth"))
    model.eval()
    sample()


import numpy as np
from nltk.stem import WordNetLemmatizer
wordnet_lemmatizer = WordNetLemmatizer()
import warnings
warnings.filterwarnings("ignore")
from sklearn.ensemble import RandomForestRegressor as RF
from sklearn.metrics.pairwise import euclidean_distances

print('Loading Data')
###################### get feature vector for train data (Y of forest fit)
with open("data/features_train/features_resnet1000intermediate_train.csv") as f:
    ncols = len(f.readline().split(','))

data = np.loadtxt(open("data/features_train/features_resnet1000intermediate_train.csv",
                       "rb"), delimiter=",", usecols=range(1,ncols))

label = np.loadtxt(open("data/features_train/features_resnet1000intermediate_train.csv"), delimiter=",", usecols=0, dtype=np.str)
labels = []
for word in label:
    labels.append((int)(word.split('/')[1].split('.')[0]))

data_label = np.concatenate((np.array(labels)[:, np.newaxis], data), axis=1)
data_label = data_label[data_label[:,0].argsort()]

label_train = data_label[:,1:]
############################

###################### get feature vector for test to match up after forest
with open("data/features_test/features_resnet1000intermediate_test.csv") as f:
    ncols = len(f.readline().split(','))

data = np.loadtxt(open("data/features_test/features_resnet1000intermediate_test.csv",
                       "rb"), delimiter=",", usecols=range(1,ncols))

label = np.loadtxt(open("data/features_test/features_resnet1000intermediate_test.csv"), delimiter=",", usecols=0, dtype=np.str)
labels = []
for word in label:
    labels.append((int)(word.split('/')[1].split('.')[0]))

data_label = np.concatenate((np.array(labels)[:, np.newaxis], data), axis=1)
data_label = data_label[data_label[:,0].argsort()]

label_test = data_label
label_test_num = label_test[:,0]
label_test = label_test[:,1:]

############################

print('Editting Data')
########################### make bag of words
#get/simplify descriptions
descriptions_train = []
for i in range(10000):
    tmp = []
    for line in open("data/descriptions_train/" + str(i) + ".txt"):
        tmp.append(line)
    descriptions_train.append(tmp)

def changeWords(s):
    #remove punctuation, but do not remove ! or ?
    punc = [':',',','-','(',')','.',';','[',']','"','*',"'",'%','&','0','1','2','3','4','5','6','7','8','9','!','?','$']
    for mark in punc:
        s = s.replace(mark,'')
    word_list = s.split(' ')
    final = []
    for word in word_list:
        #make it lowercase
        curWord = word.lower()
        curWord = curWord.strip()
        #lemmantizer
        if (curWord != 'hate'):
            curWord = wordnet_lemmatizer.lemmatize(curWord, 'v')
        #remove stop words
        stopWords = ['a','an','of','and','by','it','the','this','or','on','i',
                     'be','put','show','use','as','his','do','with','that','have','it','to',' ','for',
                     'by','up','at','those','but']
        if curWord in stopWords:
            continue
        if not (len(curWord) == 0):
            final.append(curWord)
    return " ".join(final)

tmp1 = []
for image in descriptions_train:
    tmp2 = []
    for sentence in image:
        tmp2.append(changeWords(sentence))
    tmp1.append(tmp2)
descriptions_train = tmp1

######################################

#get/simplify descriptions
descriptions_test = []
for i in range(2000):
    tmp = []
    for line in open("data/descriptions_test/" + str(i) + ".txt"):
        tmp.append(line)
    descriptions_test.append(tmp)

tmp1 = []
for image in descriptions_test:
    tmp2 = []
    for sentence in image:
        tmp2.append(changeWords(sentence))
    tmp1.append(tmp2)
descriptions_test = tmp1

######################################

print('Generating Bag of Words')
#Bag of words
dictionary = {}
for image in descriptions_train:
    for sentence in image:
        for word in sentence.split(' '):
            if word not in dictionary:
                dictionary[word] = 0

for image in descriptions_test:
    for sentence in image:
        for word in sentence.split(' '):
            if word not in dictionary:
                dictionary[word] = 0

data_train = []
for image in descriptions_train:
    tmp_dictionary = dictionary.copy()
    for sentence in image:
        for word in sentence.split(' '):
            if (word in tmp_dictionary):
                tmp_dictionary[word] = 1
    tmp_list = list(tmp_dictionary.values())
    data_train.append(tmp_list)
data_train = np.array(data_train)

keyList = list(dictionary.keys())

print('Fitting the Data to Tree...')
rf = RF(n_jobs=-1, n_estimators=20)
rf.fit(data_train, label_train)
####################################################################

data_test = []
for image in descriptions_test:
    tmp_dictionary = dictionary.copy()
    for sentence in image:
        for word in sentence.split(' '):
            if (word in tmp_dictionary):
                tmp_dictionary[word] = 1
    tmp_list = list(tmp_dictionary.values())
    data_test.append(tmp_list)
data_test = np.array(data_test)

print('Outputting Tree Predictions')
prediction = rf.predict(data_test)

def get20(vec_in):
    dist = euclidean_distances(label_test, vec_in)
    tmp = np.reshape(label_test_num, (len(label_test_num),1))
    dist_val = np.concatenate((dist, tmp), axis=1)
    dist_val_sort = dist_val[dist_val[:,0].argsort()]
    return dist_val_sort[0:50,1]

print('Getting top 20 matches')
output = []
for i in range(len(prediction)):
    out = get20(prediction[i]).astype(int)
    output.append(out) # 2000 by 20 vec

np.savetxt("features_int.csv", output, delimiter=",")

print('DONE!!!')


"""
Optimized Data structures based on arrays
"""
import itertools

class FrequencyArrays(object):
    """
    Construct a frequency array for a given array string
    """
    def __init__(self, DNA, K):
        """
        Constructs a frequency array for the given DNA and K

        :param DNA: String - DNA string
        :param K: Integer - Length of the K-mer
        """
        super(FrequencyArrays, self).__init__()
        assert K >= 1
        assert len(DNA) >= K

        self.K = K

        # All possible combinations of k-mers
        self.kmers = [''.join(p) for p in itertools.product(['A', 'C', 'T', 'G'], repeat=K)]

        # Prepare frequency array
        self.freq = [0] * pow(4, K)

        #####################
        # Precompute dictionaries of pattern_to_number and number_to_pattern
        ####################
        self.pattern_to_number = {
            kmer: self._pattern_to_number(kmer) for kmer in self.kmers
        }
        self.number_to_pattern = {
            i: self._number_to_pattern(i, K) for i in range(len(self.freq))
        }

        # compute frequency array
        for i in range(len(DNA) - K + 1):
            index = self.pattern_to_number[''.join(DNA[i : i + K])]
            self.freq[index] += 1

    def get_frequency_array(self):
        """
        Return already computed frequency array
        """
        return self.freq

    def get_most_frequent(self):
        """
        Return a list with the most frequent k-mers
        :return:
        """
        max_count = max(self.freq)
        most_freq = set()

        for i in range(len(self.kmers)):
            if self.freq[i] == max_count:
                most_freq.add(self.number_to_pattern[i, self.K])
        return most_freq

    def set_frequency(self, kmer, f):
        """
        Set f as the frequency of the kmer in the frequency array.

        This method allows the use of frequency array for sliding windows in
         a DNA string
        :param kmer: String - kmer for which to set frequency
        :param f: Integer - frequency
        """
        assert f >= 0
        self.freq[self.pattern_to_number[kmer]] = f

    def _pattern_to_number(self, kmer):
        """
        Returns the index of kmer in the frequency array.

        The approach to computing _pattern_to_number(kmer) is based on a simple
        observation. If we remove the final symbol from all lexicographically
        ordered k-mers, the resulting list is still ordered lexicographically.
        In the case of DNA strings, every (k - 1)-mer in the resulting list is
        repeated four times.

        Thus, the number of 3-mers occurring before AGT is equal to four times
        the number of 2-mers occurring before AG plus the number of 1-mers occuring
        before T. Therefore,

        _pattern_to_number(AGT) = 4 * _pattern_to_number(AG) + _symbol_to_number(T)
                                = 8 + 3 = 11
        """
        if len(kmer) == 1:
            return self._symbol_to_number(kmer)
        else:
            return 4 * self._pattern_to_number(kmer[:-1]) + self._symbol_to_number(kmer[-1])

    def _number_to_pattern(self, n, k):
        """
        Returns the k-mer that is represented by n

        In order to compute the inverse function NumberToPattern(index, k), we return to the same equation.

        The equation implies that when we divide n = _pattern_to_number(kmer) by 4,
        the remainder will be equal to _symbol_to_number(s), and the quotient will
        be equal to pattern_to_number(kmer[:]). Thus, we can use this fact to peel
        away symbols at the end of kmer one at a time.
        """
        if k == 1:
            return self._number_to_symbol(k)
        else:
            quotient, remainder = divmod(n, 4)
            return self._number_to_pattern(quotient, k-1) + self._number_to_symbol(remainder)

    def _symbol_to_number(self, symbol):
        """
        Function transforming symbols A, C, G, and T into the respective integers 0, 1, 2, and 3
        """
        nucleotides = {'A': 0, 'C': 1, 'G': 2, 'T': 3}
        return nucleotides[symbol]

    def _number_to_symbol(self, n):
        """
        Function transforming symbols o, 1, 2, and 3 into the respective nucleotides A, C, G, and T.
        """
        nucleotides = ['A', 'C', 'G', 'T']
        return nucleotides[n]


import re
import pickle
import string
import numpy as np
import pandas as pd

import spacy
from spacy.lang.en.stop_words import STOP_WORDS
from spacy.lang.en import English
from spacy import displacy, lemmatizer

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split

# Clean text so it's ready for preprocessing

def clean(corpus):
    """
    Takes a speech as an argument and cleans it so it's ready to be pre-processed
    """
    # Initiate clean_corpus
    clean_corpus = []

    for speech in corpus:

        # Removes meaningless intro
        speech = speech[5:]

        for i in range(len(speech)):
            # Removes 'meaningless text hear (min:sec)\n' at the beginning of each paragraph
            speech[i] = speech[i][speech[i].find('\n') + 1:]
            # Replaces brackets with paranthesis
            speech[i] = speech[i].replace('[', '(')
            speech[i] = speech[i].replace(']', ')')
            # Removes meaningless text in parantheses
            speech[i] = re.sub(r'\([^)]*\)', '', speech[i])

        # Join all of the paragraphs into one speech
        speech = ','.join(speech)

        clean_corpus.append(speech)

    # Combined all of the speeches into one document

    if len(clean_corpus) == 1:
        clean_corpus = clean_corpus[0]
    if len(clean_corpus) == 2:
        clean_corpus = clean_corpus[0] + clean_corpus[1]
    if len(clean_corpus) == 3:
        clean_corpus = clean_corpus[0] + clean_corpus[1] + clean_corpus[2]
    if len(clean_corpus) == 8:
        clean_corpus = clean_corpus[0] + clean_corpus[1] + clean_corpus[2] + clean_corpus[3] + clean_corpus[4] + \
                       clean_corpus[5] + clean_corpus[6] + clean_corpus[7]

    return clean_corpus

# Tokenize corpus and preprocess text

def preprocess(clean_corpus):
    """
    Takes in a speech, tokenizes and breaks it out into sentences, lemmatizes and removes stop words and punctuation for each speech
    """

    # Create our list of punctuation marks
    punctuations = '!"#$%&\'()’*+,-./:”;<=>?@[\\]^_`{|}~'

    # Create our list of stopwords
    stop_words = spacy.lang.en.stop_words.STOP_WORDS

    # Establish weird words in our corpus to be filtered out
    weird_words = ['abc', 'c', 'o', 'n','']

    # Etablish new sentence list
    sentence_list = []

    # Tokenizer
    tokenizer = spacy.load('en_core_web_sm')
    corpus_tokens = tokenizer(clean_corpus)

    for sentence in corpus_tokens.sents:

        # Lemmatizing each token and converting each token into lowercase
        sentence = [word.lemma_.lower().strip() if word.lemma_ != "-PRON-" else word.lower_ for word in sentence]

        # Remove stop words, weird words and punctuations
        sentence = [word for word in sentence if word not in stop_words and word not in punctuations \
                         and word not in weird_words]

        # Remove numbers
        sentence = [word for word in sentence if word.isalpha()]

        # Join all words in sentence
        sentence = ','.join(sentence)

        # Append to new list
        sentence_list.append(sentence)

    # Remove empty sentences
    sentence_list = [sentence for sentence in sentence_list if sentence != '']

    return sentence_list


from urllib import request,parse

headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}

def search(params):
    url = 'https://www.baidu.com/s?'+params
    req = request.Request(url=url, headers=headers)
    res = request.urlopen(req)
    return res.read().decode()

if __name__ == '__main__':
    param = input("请输入您要查找的内容:")
    wd = {
        "wd":param
    }
    params = parse.urlencode(wd)
    print(search(params))


from donkeycar.parts.controller import Joystick, JoystickController

class MyJoystick(Joystick):
    #An interface to a physical joystick available at /dev/input/js0
    def __init__(self, *args, **kwargs):
        super(MyJoystick, self).__init__(*args, **kwargs)

        self.button_names = {
            0x136 : 'LB',
            0x137 : 'RB',
            0x134 : 'Y',
            0x131 : 'B',
            0x130 : 'A',
            0x133 : 'X',
            0x13a : 'BACK',
            0x13b : 'START',
            0x13c : 'HOME',
            0x13d : 'LSB',
            0x13e : 'RSB',
        }

        self.axis_names = {
            0x1 : 'LVERT',
            0x0 : 'LHORIZ',
            0x4 : 'RVERT',
            0x3 : 'RHORIZ',
            0x2 : 'LT',
            0x5 : 'RT',
            0x10 : 'CROSSHORIZ',
            0x11 : 'CROSSVERT',
        }

class MyJoystickController(JoystickController):
    #A Controller object that maps inputs to actions
    def __init__(self, *args, **kwargs):
        super(MyJoystickController, self).__init__(*args, **kwargs)

    def init_js(self):
        #attempt to init joystick
        try:
            self.js = MyJoystick(self.dev_fn)
            self.js.init()
        except FileNotFoundError:
            print(self.dev_fn, "not found.")
            self.js = None
        return self.js is not None

    def init_trigger_maps(self):
        #init set of mapping from buttons to function calls

        self.button_down_trigger_map = {
            'A' : self.erase_last_N_records,
            'B' : self.emergency_stop,
            'LB' : self.toggle_mode,
        }

        self.axis_trigger_map = {
            'RHORIZ' : self.set_steering,
            'LVERT' : self.set_throttle,
        }


import cv2 as ocv
import time
from bluedot import BlueDot
from signal import pause

camera = ocv.VideoCapture(0)
webcam = 0
picture = 0

camera.set(3, 640) #Largeur
camera.set(4, 480) #Hauteur
camera.set(10, 200) #Luminosite

fermion = ocv.imread("Ressources/Fermion.jpg")

def trigger(position):

    if position.top:

        print("Starting Webcam")
        webcam = 1

    elif position.bottom:

        print("Picture Display")
        ocv.imshow("Fermion", fermion)
        time.sleep(5)

    elif position.right:

        print("STOP")
        exit()

bd = BlueDot()

bd.when_pressed = trigger

pause()

Programmes et ressouces de tests/TESTOCV.py


#!/usr/bin/python
#
#   Copyright 2009 Rev. Johnny Healey
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#

from julep import master
from julep import config

import optparse
import sys

def main(config_file,pid_file):
	master_server = master.JulepMasterServer()
	master_server.load_config(config_file)
	master_server.daemonize(pid_file)

def testconfig(config_file):
	config.test_config(config_file)

if __name__ == '__main__':
	parser = optparse.OptionParser()

	parser.add_option("-c","--config",dest="config_file",
		help="the location of the configuration file",
		metavar="FILE",default="/etc/julep/config.py")

	parser.add_option("-p","--pid-file",dest="pid_file",
		help="the location of the pid file created by the daemon",
		metavar="FILE",default="/var/run/julep.pid")

	parser.add_option("-t","--test-config",dest="test_config",
		help="test the validity of the configuration file then exit.",
		action="store_true",default=False)

	options,args = parser.parse_args()

	if options.test_config:
		if config.test_config(options.config_file):
			sys.exit(0)
		else:
			sys.exit(1)

	main(options.config_file,options.pid_file)


import paddle.nn as nn

class ConvHead(nn.Layer):
    def __init__(self, out_channels, in_channels=32, num_layers=1,
                 kernel_size=3, padding=1,
                 norm_layer=nn.BatchNorm2D):
        super(ConvHead, self).__init__()
        convhead = []

        for i in range(num_layers):
            convhead.extend([
                nn.Conv2D(in_channels, in_channels, kernel_size, padding=padding),
                nn.ReLU(),
                norm_layer(in_channels)
            ])
        convhead.append(nn.Conv2D(in_channels, out_channels, 1, padding=0))

        self.convhead = nn.Sequential(*convhead)

    def forward(self, *inputs):
        return self.convhead(inputs[0])

class SepConvHead(nn.Layer):
    def __init__(self, num_outputs, in_channels, mid_channels, num_layers=1,
                 kernel_size=3, padding=1, dropout_ratio=0.0, dropout_indx=0,
                 norm_layer=nn.BatchNorm2D):
        super(SepConvHead, self).__init__()

        sepconvhead = []

        for i in range(num_layers):
            sepconvhead.append(
                SeparableConv2d(in_channels=in_channels if i == 0 else mid_channels,
                                out_channels=mid_channels,
                                dw_kernel=kernel_size, dw_padding=padding,
                                norm_layer=norm_layer)
            )
            if dropout_ratio > 0 and dropout_indx == i:
                sepconvhead.append(nn.Dropout(dropout_ratio))

        sepconvhead.append(
            nn.Conv2D(in_channels=mid_channels, out_channels=num_outputs, kernel_size=1, padding=0)
        )

        self.layers = nn.Sequential(*sepconvhead)

    def forward(self, *inputs):
        x = inputs[0]

        return self.layers(x)

class SeparableConv2d(nn.Layer):
    def __init__(self, in_channels, out_channels, dw_kernel, dw_padding, dw_stride=1, use_bias=False, norm_layer=None):
        super(SeparableConv2d, self).__init__()
        self.body = nn.Sequential(
            nn.Conv2D(in_channels, in_channels, kernel_size=dw_kernel, stride=dw_stride,
                      padding=dw_padding, bias_attr=use_bias, groups=in_channels),
            nn.Conv2D(in_channels, out_channels, kernel_size=1, stride=1, bias_attr=use_bias),
            norm_layer(out_channels),
            nn.ReLU()
        )

    def forward(self, x):
        return self.body(x)


from setuptools import setup

entrypoints = {
        'console_scripts': ['HydXS=HydXS.cli_interface:app'],
    }

setup(name='HydXS',
      version='0.1a',
      description='A tool for automatic detection of bankfull boundaries from river cross section data',
      url='http://github.com/sailngarbwm/HydXS',
      author='kathryn russell',
      author_email='kathryn.russel@unimelb.edu.au',
      license='CC',
      packages=['HydXS'],
      entry_points = entrypoints,
      zip_safe=False)


# -*- coding: utf-8 -*-
"""
Created on Mon Aug 24 08:32:41 2020

@author: Dr~Newt
"""

import numpy as np
import cv2
from utils5.image_Filter import ImageFilters
from utils5.projection_Manager import Projection_Mgr
from utils5.lane import Lane
from utils5.line import Line
import sys

class RoadManager():
    # Initialize roadManager

    def __init__(self, camCal):
        self.current_Frame = None
        self.mtx, self.dist, self.img_size = camCal.get()
        self.x, self.y = self.img_size
        self.mid = int(self.y / 2)
        self.left_a, self.left_b, self.left_c = [], [], []
        self.right_a, self.right_b, self.right_c = [], [], []
        self.im_filt = ImageFilters(camCal)
        self.Final_proc_img = None
    #*******************************************************************************************#
    #                          TRACK BACK INTERFACE DESIGN                                      #
    #*******************************************************************************************#
    def nothing(x):# call back function for tackbar : does nothing
        pass

    def initializeTrackbars(self, init_Tracbar_Vals):#track bar interface

        cv2.namedWindow("Trackbars")
        cv2.resizeWindow("Trackbars", 360, 240)
        cv2.createTrackbar("Width Top", "Trackbars", init_Tracbar_Vals[0],50, self.nothing)
        cv2.createTrackbar("Height Top", "Trackbars", init_Tracbar_Vals[1], 100, self.nothing)
        cv2.createTrackbar("Width Bottom", "Trackbars", init_Tracbar_Vals[2], 50, self.nothing)
        cv2.createTrackbar("Height Bottom", "Trackbars", init_Tracbar_Vals[3], 100, self.nothing)

    def valTrackbars(self):#Tracks the track bar position

        width_Top = cv2.getTrackbarPos("Width Top", "Trackbars")
        height_Top = cv2.getTrackbarPos("Height Top", "Trackbars")
        width_Bottom = cv2.getTrackbarPos("Width Bottom", "Trackbars")
        height_Bottom = cv2.getTrackbarPos("Height Bottom", "Trackbars")

        src_points = np.float32([(width_Top/100,height_Top/100), (1-(width_Top/100), height_Top/100),
                          (width_Bottom/100, height_Bottom/100), (1-(width_Bottom/100), height_Bottom/100)])
        return src_points

    def drawPoints(self, img,src_points):#Draws points on image to define prespective warp

        img_size = np.float32([(img.shape[1],img.shape[0])])
        src_points = src_points * img_size
        for x in range( 0,4):
            cv2.circle(img,(int(src_points[x][0]),int(src_points[x][1])),15,(0,0,255),cv2.FILLED)
        return img

    def textDisplay(self, img,curvature, frame):
        # num_Vehicle = Projection_Mgr()
        font_Type = cv2.FONT_HERSHEY_COMPLEX
        cv2.putText(img, "Curvature : " +str(curvature), ((img.shape[1]//2)+100, 35), font_Type, 0.4, (0, 255, 255), 1, cv2.LINE_AA)
        cv2.putText(img, "Frame Number : " + str(frame), ((img.shape[1]//2)+100, 15), font_Type, 0.4, (0, 255, 255), 1, cv2.LINE_AA)
        cv2.putText(img, "Vehicle Count : " + str("num_Vehicle.Num_Vehicle"), ((img.shape[1]//2)+100, 55), font_Type, 0.4, (0, 255, 255), 1, cv2.LINE_AA)
        cv2.putText(img, "Pedestrian : " + str(0), ((img.shape[1]//2)+100, 75), font_Type, 0.4, (0, 255, 255), 1, cv2.LINE_AA)
        cv2.rectangle(img, (0, 20), (200, 0),(255,255,255), 30 )
        direction_Text = "No lanes Detected"

        if curvature > 10:
            direction_Text = "Turn Right"
        elif curvature < -10:
            direction_Text = "Turn Left"
        elif curvature >= 4 and curvature < 10 or curvature >= -4 and curvature < -10 :
            direction_Text = "Go Straight"
        elif curvature == -1000000:
            cv2.putText(img, "WARNING !!!", ((img.shape[1]//2)-35, 205 ), font_Type, 0.8, (0, 0, 255), 1, cv2.LINE_AA)
            direction_Text = "No lanes Detected"

        cv2.putText(img,direction_Text, ((img.shape[1]//2)-35,205 ), font_Type, 0.5, (0, 0, 255), 1, cv2.LINE_AA)

    #Returns the birds eye view of the image(frame)
    def perspective_warp(self, img, dst_size=(1280, 720),
                         src_points = np.float32([(0.43,0.65),(0.58,0.65),(0.1,1),(1,1)]),
                         dst=np.float32([(0,0), (1, 0), (0,1), (1,1)])):

        img_size = np.float32([(img.shape[1],img.shape[0])])
        src_points = src_points* img_size
        dst = dst * np.float32(dst_size)
        # Given src and dst points, calculate the perspective transform matrix
        M = cv2.getPerspectiveTransform(src_points, dst)
        warped = cv2.warpPerspective(img, M, dst_size)
        return warped

    #returns an image
    def inv_perspective_warp(self, img, dst_size=(1280,720),
                         src_points=np.float32([(0,0), (1, 0), (0,1), (1,1)]),
                         dest_points=np.float32([(0.43,0.65),(0.48,0.65),(0.1,1),(1,1)])):

        img_size = np.float32([(img.shape[1],img.shape[0])])
        src_points = src_points* img_size
        dest_points = dest_points * np.float32(dst_size)
        presp_mtx = cv2.getPerspectiveTransform(src_points, dest_points)#calculate the perspective transform matrix
        inv_warped = cv2.warpPerspective(img, presp_mtx, dst_size)
        return inv_warped

    def get_hist(self, img):
        hist = np.sum(img[img.shape[0]//2:,:], axis=0)
        return hist

    def sliding_window(self, img, nwindows=15, margin=60, minpix=1, draw_windows=True):

        #for storing a,b and c after computation
        current_left_fit = np.empty(3)
        current_right_fit = np.empty(3)
        # out_img = img
        out_img = np.dstack((img,img,img))*255
        histogram = self.get_hist(img)
        # find peaks of left and right halves from hist
        midpoint = int(histogram.shape[0] / 2)
        leftx_base = np.argmax(histogram[:midpoint])
        rightx_base = np.argmax(histogram[midpoint:]) + midpoint
        # Set height of windows
        window_height = np.int(img.shape[0] / nwindows)
        # for Identifying the x and y positions of all non zero pixels in the binary image
        nonzero = img.nonzero()
        nonzeroy = np.array(nonzero[0])
        nonzerox = np.array(nonzero[1])
        # Current positions to be updated for each window
        leftx_current = leftx_base
        rightx_current = rightx_base
        # Create empty lists to receive left and right lane pixel indices
        left_lane_inds = []
        right_lane_inds = []
        # Step through the windows one by one
        for window in range(nwindows):
            # Identify window boundaries in x and y (and right and left)
            win_y_low = img.shape[0] - (window + 1) * window_height
            win_y_high = img.shape[0] - window * window_height
            win_xleft_low = leftx_current - margin
            win_xleft_high = leftx_current + margin
            win_xright_low = rightx_current - margin
            win_xright_high = rightx_current + margin
            # Draw the windows or rectangles on the birds eye view binary image
            if draw_windows == True:
                cv2.rectangle(out_img, (win_xleft_low, win_y_low), (win_xleft_high, win_y_high),(0, 0, 255), 2)
                cv2.rectangle(out_img, (win_xright_low, win_y_low), (win_xright_high, win_y_high),(255, 0, 0), 2)
            # Identify the nonzero pixels in x and y within the window
            good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) &
                              (nonzerox >= win_xleft_low) & (nonzerox < win_xleft_high)).nonzero()[0]

            good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) &
                               (nonzerox >= win_xright_low) & (nonzerox < win_xright_high)).nonzero()[0]

            # Append these indices to the lists
            left_lane_inds.append(good_left_inds)
            right_lane_inds.append(good_right_inds)

            # If you found > minpix pixels, recenter next window on their mean position
            if len(good_left_inds) > minpix:
                leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
            if len(good_right_inds) > minpix:
                rightx_current = np.int(np.mean(nonzerox[good_right_inds]))

        # Concatenate the arrays of indices
        left_lane_inds = np.concatenate(left_lane_inds)
        right_lane_inds = np.concatenate(right_lane_inds)

        # Extract left and right line pixel positions
        leftx = nonzerox[left_lane_inds]
        lefty = nonzeroy[left_lane_inds]
        rightx = nonzerox[right_lane_inds]
        righty = nonzeroy[right_lane_inds]

        if leftx.size and rightx.size:

            # Fit a second order polynomial to to left and right lane pixel values
            left_fit = np.polyfit(lefty, leftx, 2)
            right_fit = np.polyfit(righty, rightx, 2)

            #captures the calculated values of a,b and c for right and left
            self.left_a.append(left_fit[0])
            self.left_b.append(left_fit[1])
            self.left_c.append(left_fit[2])
            self.right_a.append(right_fit[0])
            self.right_b.append(right_fit[1])
            self.right_c.append(right_fit[2])

            current_left_fit[0] = np.mean(self.left_a[-10:])
            current_left_fit[1] = np.mean(self.left_b[-10:])
            current_left_fit[2] = np.mean(self.left_c[-10:])

            current_right_fit[0] = np.mean(self.right_a[-10:])
            current_right_fit[1] = np.mean(self.right_b[-10:])
            current_right_fit[2] = np.mean(self.right_c[-10:])

            # Generate x and y values for plotting
            ploty = np.linspace(0, img.shape[0] - 1, img.shape[0])

            left_fitx_eq = current_left_fit[0] * ploty ** 2 + current_left_fit[1] * ploty + current_left_fit[2]
            right_fitx_eq = current_right_fit[0] * ploty ** 2 + current_right_fit[1] * ploty + current_right_fit[2]

            out_img[nonzeroy[left_lane_inds], nonzerox[left_lane_inds]] = [255, 0, 100]
            out_img[nonzeroy[right_lane_inds], nonzerox[right_lane_inds]] = [0, 100, 255]

            return out_img, (left_fitx_eq, right_fitx_eq), (current_left_fit, current_right_fit), ploty
        else:
            return img,(0,0),(0,0),0

    def get_curve(self, img, leftx, rightx):

        ploty = np.linspace(0, img.shape[0] - 1, img.shape[0])
        y_eval = np.max(ploty)
        ym_per_pix = 1 / img.shape[0]  # meters per pixel in y dimension
        xm_per_pix = 0.1 / img.shape[0]  # meters per pixel in x dimension

        # Fit new polynomials to x,y in world space
        left_fit_cr = np.polyfit(ploty * ym_per_pix, leftx * xm_per_pix, 2)
        right_fit_cr = np.polyfit(ploty * ym_per_pix, rightx * xm_per_pix, 2)
        # Calculate the new radii of curvature
        left_curverad = ((1 + (2 * left_fit_cr[0] * y_eval * ym_per_pix + left_fit_cr[1]) ** 2) ** 1.5) / np.absolute(
            2 * left_fit_cr[0])
        right_curverad = ((1 + (2 * right_fit_cr[0] * y_eval * ym_per_pix + right_fit_cr[1]) ** 2) ** 1.5) / np.absolute(
            2 * right_fit_cr[0])

        car_pos = img.shape[1] / 2
        l_fit_x_int = left_fit_cr[0] * img.shape[0] ** 2 + left_fit_cr[1] * img.shape[0] + left_fit_cr[2]
        r_fit_x_int = right_fit_cr[0] * img.shape[0] ** 2 + right_fit_cr[1] * img.shape[0] + right_fit_cr[2]
        lane_center_position = (r_fit_x_int + l_fit_x_int) / 2
        center = (car_pos - lane_center_position) * xm_per_pix / 10
        # Now our radius of curvature is in meters

        return (l_fit_x_int, r_fit_x_int, center)

    def draw_lanes(self, img, left_fit, right_fit,frameWidth,frameHeight,src):

        ploty = np.linspace(0, img.shape[0]-1 , img.shape[0])
        color_img = np.zeros_like(img)

        left = np.array([np.transpose(np.vstack([left_fit, ploty]))])
        right = np.array([np.flipud(np.transpose(np.vstack([right_fit, ploty])))])
        points = np.hstack((left, right + 15))

        cv2.fillPoly(color_img, np.int_(points), (0, 150, 0))
        inv_perspective = self.inv_perspective_warp(color_img,(frameWidth,frameHeight),dest_points = src)
        inv_perspective = cv2.addWeighted(img, 0.5, inv_perspective, 0.7, 0)
        return inv_perspective

    def Hough_Lines_Draw(self, img):

        lines = cv2.HoughLinesP(img ,2, np.pi/180, 100 , np.array([]), minLineLength=40, maxLineGap = 5)
        mask = np.zeros((self.y, self.x, 3), dtype=np.float32)
        if lines is not None:
            for line in lines:
                x1, y1, x2, y2 = line.reshape(4)
                cv2.line(mask, (x1,y1), (x2,y2), (255,255,255),10)
            return mask
        return None

    def drawLines(self, img,lane_curve):#Draws vertical lines
        myWidth = img.shape[1]
        myHeight = img.shape[0]
        for x in range(-30, 30):
            w = myWidth // 20
            cv2.line(img, (w * x + int(lane_curve // 100), myHeight - 30),
                     (w * x + int(lane_curve // 100), myHeight), (0, 0, 255), 2)
        cv2.line(img, (int(lane_curve // 100) + myWidth // 2, myHeight - 30),
                 (int(lane_curve // 100) + myWidth // 2, myHeight), (0, 255, 0), 3)
        cv2.line(img, (myWidth // 2, myHeight - 50), (myWidth // 2, myHeight), (255, 0, 0), 2)

        return img

    #Combines images and shows the main pipeline
    def stackImages(self, scale,imgArray):
        rows = len(imgArray)
        cols = len(imgArray[0])
        rowsAvailable = isinstance(imgArray[0], list)
        width = imgArray[0][0].shape[1]
        height = imgArray[0][0].shape[0]
        if rowsAvailable:
            for x in range ( 0, rows):
                for y in range(0, cols):

                    if imgArray[x][y].shape[:2] == imgArray[0][0].shape [:2]:
                        imgArray[x][y] = cv2.resize(imgArray[x][y], (0, 0), None, scale, scale)
                    else:
                        imgArray[x][y] = cv2.resize(imgArray[x][y], (imgArray[0][0].shape[1], imgArray[0][0].shape[0]), None, scale, scale)
                    if len(imgArray[x][y].shape) == 2: imgArray[x][y]= cv2.cvtColor( imgArray[x][y], cv2.COLOR_GRAY2BGR)
            imageBlank = np.zeros((height, width, 3), np.uint8)
            hor = [imageBlank]*rows
            hor_con = [imageBlank]*rows
            for x in range(0, rows):
                hor[x] = np.hstack(imgArray[x])
            ver = np.vstack(hor)
        else:
            for x in range(0, rows):
                if imgArray[x].shape[:2] == imgArray[0].shape[:2]:
                    imgArray[x] = cv2.resize(imgArray[x], (0, 0), None, scale, scale)
                else:
                    imgArray[x] = cv2.resize(imgArray[x], (imgArray[0].shape[1], imgArray[0].shape[0]), None,scale, scale)
                if len(imgArray[x].shape) == 2: imgArray[x] = cv2.cvtColor(imgArray[x], cv2.COLOR_GRAY2BGR)
            hor= np.hstack(imgArray)
            ver = hor
        return ver


from flask import Flask
# from forms.loginform import LoginForm
# from forms.registerform import RegisterForm
import db
import json

from flask import request
from flask import send_from_directory
from flask import send_file

db.main()
app = Flask(__name__)
app.config['SECRET_KEY'] = 'qwekwqkJDHASIqwop'

@app.route('/download/database')
def download():
    # return "ppp"
    return send_file("db/gideTMN55.db", as_attachment=True, attachment_filename="gideTMN55.db", )
    # return send_from_directory('db/', "gideTMN55.db")

@app.route('/', methods=['GET', 'POST'])
def parse_request():
    data = request.args
    data_json = request.json
    print("/////////")
    print(data)
    print(data_json)
    return str(data_json)

@app.route('/hi', methods=['GET', 'POST'])
def _hi():
    return 'Hi angry world! '

@app.route('/questtours', methods=['GET', 'POST'])
def _questtours():
    # data = request.data
    db_sess = db.db_session.create_session()
    data = db.get_questtours(db_sess)
    print("questtours:", data)
    return json.dumps(data)

@app.route('/create_questtour', methods=['GET', 'POST'])
def _create_questtour():
    data_json = request.json
    db_sess = db.db_session.create_session()
    data = db.create_questtour(db_sess, **data_json)
    print("create_questtour:", data)
    return json.dumps({"id": data.id})

@app.route('/create_questtours', methods=['GET', 'POST'])
def _create_questtours():
    data_json = request.json
    return json.dumps(db._create_tests(data_json))

@app.route('/create_test', methods=['GET', 'POST'])
def _create_test():
    data_json = request.json
    db_sess = db.db_session.create_session()
    data = db.create_test(db_sess, **data_json)
    print("create_test:", data)
    return json.dumps({"id": data.id})

@app.route('/create_tests', methods=['GET', 'POST'])
def _create_tests():
    data_json = request.json
    return json.dumps(db._create_tests(data_json))

@app.route('/create_vidiotour', methods=['GET', 'POST'])
def _create_vidiotour():
    data_json = request.json
    db_sess = db.db_session.create_session()
    data = db.create_vidiotour(db_sess, **data_json)
    print("create_vidiotour:", data)
    return json.dumps({"id": data.id})

@app.route('/create_vidiotours', methods=['GET', 'POST'])
def _create_vidiotours():
    data_json = request.json
    return json.dumps(db._create_vidiotours(data_json))

@app.route('/vidiotours', methods=['GET', 'POST'])
def _vidiotours():
    db_sess = db.db_session.create_session()
    data = db.get_vidiotours(db_sess)
    print("vidiotours:", data)
    return json.dumps(data)

@app.route('/tests', methods=['GET', 'POST'])
def _tests():
    tour_id = request.args["tour_id"]
    db_sess = db.db_session.create_session()
    data = db.get_tests(db_sess, tour_id)
    print("tests", data)
    return json.dumps(data)

if __name__ == "__main__":
    app.run(debug=False)


#coding: utf-8

from threading import Thread
import shell

import socket
import threading

DEFALUT_PORT = 12345

def main():

    port = set_port()

    if not port: return 0

    Server(port).start()

    return 0

def set_port():
    port = input('PORT[%d]: ' % DEFALUT_PORT)
    print()

    if (port == ''):
        port = DEFALUT_PORT
    else:
        try:
            port = int(port)
        except ValueError:
            print ('Porta inválida.')
            return

    return port

class Server:

    def __init__(self, port):
        self.__socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.__socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.__socket.bind(('', port))
        self.__socket.listen(5)

        self.__connections = []
        self.__lock = threading.Lock()

    def start(self):
        try:
            Thread(target=self.__accept, args=()).start()

            print('\nO shell remoto está aceitando conexões.\nPara finalizar digite "exit" ou use "Ctrl+C"\n\n')

            exit_flag = input()

            while (exit_flag != 'exit'):
                exit_flag = input()
        except KeyboardInterrupt:
            pass
        finally:
            self.close()

    def __accept(self):
        while(True):
            connection, client = self.__socket.accept()
            Thread(target=self.newConnection, args=(connection, client)).start()

    def newConnection(self, connection, client):
        import os

        try:
            directory = os.getcwd() + '/'

            print ('Conectado com %s na porta %d ' % client)

            with self.__lock:
                self.__connections.append(connection)

            while (True):
                command = connection.recv(1024).decode()

                if (command == '_exit'): break

                if (command == '_start'):
                    connection.send((socket.gethostname() + ':' + directory).encode())
                    continue

                output = None
                erro = None

                output, erro, directory  = shell.executar_comando(command, directory)

                if (erro):
                    self.__send(erro.decode(), connection, directory)
                elif (output):
                    self.__send(output.decode(), connection, directory)
                else:
                    self.__send(' ', connection, directory)
        finally:
            with self.__lock:
                self.__connections.remove(connection)
                print ('Finalizando conexao com o cliente %s na porta %d  ' % connection.getpeername())
                connection.close()

    @staticmethod
    def __send(data, socket_, directory):
        socket_.send(('%s:%s[-.x.-]%s[-.x.-]%s' % (socket.gethostname(), directory, len(data), data)).encode())

    def close(self):
        if (self.__socket):
            if (len(self.__connections) > 0):
                with self.__lock:
                    print('\nEncerrando conexoes...\n')
                    for connection in self.__connections:
                        print ('Finalizando conexao com o cliente %s na porta %d  ' % connection.getpeername())
                        connection.close()
            self.__socket.close()
        print()

if __name__ == "__main__":
    import os
    os._exit(main())


import numpy as np
import tables
from datetime import datetime
startTime = datetime.now()
version = "9"

def getSeqListFromH5File(filename= "..//outputfile//"+version+"//info.h5"):
    infoFile = tables.open_file(filename, mode = 'r')
    seqList = infoFile.root.sequences.read()
    infoFile.close()
    return seqList

def getListFeature(seqList, thresholdFreq=0, sizeOfSubstr = range(2,41)):
    dictSubStrFrequency ={}
    for sizeOfSub in sizeOfSubstr:
        for seq in seqList:
            sizeOfSeq = len(seq)
            for index in xrange(0,sizeOfSeq-sizeOfSub+1):
                subSeq = seq[index: index+sizeOfSub]
                if subSeq in dictSubStrFrequency: #count so lan xuat hien sub subStr
                    dictSubStrFrequency[subSeq] += 1
                else:
                    dictSubStrFrequency[subSeq] = 1

    listFeature = filter(lambda key: dictSubStrFrequency[key] > thresholdFreq, dictSubStrFrequency) #lay cac subSeq co tan so xuat hien > thresholdFreq
    return listFeature

#build relational matrix
def buildRelationalMatrixToH5File(seqList, listFeature, filename = "..//outputfile//"+version+"//encodingFile.h5"):
    fileh = tables.open_file(filename, mode="w")
    # Lay root cua file
    root = fileh.root
    sizeOfSeqList =  len(seqList)

    atom = tables.Int16Atom()

    featureRowMatrix = fileh.create_earray(root, 'featureRowMatrix', atom, (0,sizeOfSeqList),"featureRowMatrix")
    seqRowMatrix = fileh.create_earray(root, 'seqRowMatrix', atom, (sizeOfSeqList,0),"seqRowMatrix")

    for curFeatureIndex in range(len(listFeature)):
        print (curFeatureIndex)
        curFeature = listFeature[curFeatureIndex]
        arr = np.zeros((sizeOfSeqList,), np.uint16)

        #dem so lan xuat hien
        for indexOfSeq in xrange(sizeOfSeqList):
            arr[indexOfSeq] = seqList[indexOfSeq].count(curFeature)

        # luu array vao HDF5 file
        featureRowMatrix.append([arr])
        seqRowMatrix.append(arr.reshape(sizeOfSeqList,1))
    fileh.flush()
    fileh.close()

"""
tao tuple gom index cac array giong array dang xet
co chua luon index cua array dang xet
muc dich de remove duplicate feature Vecto
ham bo tro cho groupDuplicateFeatureRow
"""
def getGroupIndexOfSameArrInNdarray(matrix, indexArr, listSubStr):
    listRet = []
    numOfRowMat = len(matrix)
    for curIndex in range(numOfRowMat):
        if np.array_equal(matrix[curIndex],matrix[indexArr]):
            if (listSubStr[curIndex] in listSubStr[indexArr]) or (listSubStr[indexArr] in listSubStr[curIndex]):
                listRet.append(curIndex)

    #chi chua 1 index duy nhat la chinh no ==> khong co arr nao giong no
    if (len(listRet) == 1):
        return None
    else:
        return listRet

#remove duplicate feature vector
def isNumIn2DList(num, list2DNum):
    for arr in list2DNum:
        for curNum in arr:
            if curNum == num:
                return True
    return False

#group cac feature co vector feature giong nhau
def getGroupDuplicateFeatureRow(listFeature, featureRowMatrix):
    groupDuplicateFeatureRow = []
    for curFeatureIndex in range(len(listFeature)):
        print (curFeatureIndex)
        # da nam trong 1 group duplicate nao do khong can xet tiep
        if isNumIn2DList(curFeatureIndex,groupDuplicateFeatureRow):
            pass
        else:
            #check xem co bi duplicate k
            groupDupIndex = getGroupIndexOfSameArrInNdarray(featureRowMatrix,curFeatureIndex, listFeature)
            if groupDupIndex != None: # bi duplicate
                groupDuplicateFeatureRow.append(groupDupIndex)
    return groupDuplicateFeatureRow

"""
tim index cua substring co length lon nhat trong cac substr co index trong listIndexSubStr
"""
def getIndexOfLongestStrInListIndex(listIndexSubStr, listAllSubStr):
    maxLength = -1
    retIndex = -1
    for curIndexSubStr in listIndexSubStr:
        curLength = len(listAllSubStr[curIndexSubStr])
        if  curLength > maxLength:
            maxLength = curLength
            retIndex = curIndexSubStr
    return retIndex

#lap list index cua cac feature duplicate can loai bo
def getListDuplicateFeatureRowIndex(listFeature,groupDuplicateFeatureRow):
    listDuplicateFeatureRowIndex =[]
    for group in groupDuplicateFeatureRow:
        #lay feature co do dai lon nhat, bo cac feature la con cua no khi cac vector giong nhau
        selectedFeatureIndex = getIndexOfLongestStrInListIndex(group,listFeature)
        group.remove(selectedFeatureIndex)
        listDuplicateFeatureRowIndex = listDuplicateFeatureRowIndex + group
    return listDuplicateFeatureRowIndex

#remove feature vecto duplicate
def removeDuplicateFeatureVec(featureRowMatrix, listFeature, listDuplicateFeatureRowIndex):
    newFeatureRowMatrix = np.delete(featureRowMatrix,listDuplicateFeatureRowIndex,axis = 0)
    newListFeature= np.delete(np.array(listFeature),listDuplicateFeatureRowIndex,axis = 0)
    return newFeatureRowMatrix, newListFeature

#luu relational table moi thay the cai da co
def replaceRelationalMatFromH5File(newFeatureRowMatrix, filename = "..//outputfile//"+version+"//encodingFile.h5"):
    fileh = tables.open_file(filename, mode="a")
    root = fileh.root
    root.featureRowMatrix._f_remove()
    root.seqRowMatrix._f_remove()
    fileh.create_array(root, 'featureRowMatrix', newFeatureRowMatrix,"featureRowMatrix")
    fileh.create_array(root, 'seqRowMatrix', np.transpose(newFeatureRowMatrix),"seqRowMatrix")
    fileh.flush()
    fileh.close()

def saveInfoFeatureToH5File(listFeature,sizeOfSubstr=range(2,41), filename = "..//outputfile//"+version+"//encodingFile.h5"):
    fileh = tables.open_file(filename, mode="a")
    root = fileh.root
    fileh.create_array(root, 'feature', listFeature, 'feature')
    fileh.create_array(root, 'nOfNgram', sizeOfSubstr, 'nOfNgram')
    fileh.flush()
    fileh.close()

def encode():
    #get feature
    sizeOfSubstr = range(2,41)
    seqList = getSeqListFromH5File()
    listFeature = getListFeature(seqList,thresholdFreq=0.5*len(seqList), sizeOfSubstr= sizeOfSubstr)

    #build relational Matrix & save to H5 file
    buildRelationalMatrixToH5File(seqList,listFeature)

    #remove duplicate feature Row
    fileh = tables.open_file("..//outputfile//"+version+"//encodingFile.h5", mode="r")
    featureRowMatrix =fileh.root.featureRowMatrix
    groupDuplicateFeatureRow = getGroupDuplicateFeatureRow(listFeature, featureRowMatrix)

    listDuplicateFeatureRowIndex = getListDuplicateFeatureRowIndex(listFeature,groupDuplicateFeatureRow)
    newFeatureRowMatrix, newListFeature = removeDuplicateFeatureVec(featureRowMatrix,listFeature,listDuplicateFeatureRowIndex)
    fileh.close()

    #save info again
    replaceRelationalMatFromH5File(newFeatureRowMatrix)
    saveInfoFeatureToH5File(newListFeature,sizeOfSubstr)

if __name__ == "__main__":
    encode()


import pandas as pn
from sklearn.linear_model import Perceptron
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import StandardScaler

data_train = pn.read_csv('/Users/oleg/PycharmProjects/ML-Found-Yandex/DATA/train.csv', header = None)
data_test = pn.read_csv('/Users/oleg/PycharmProjects/ML-Found-Yandex/DATA/test.csv', header = None)

clf = Perceptron(random_state=241)
clf.fit(data_train.loc[:, 1:], data_train.loc[:, 0])
predictions = clf.predict(data_test.loc[:, 1:])
accuracy_before = accuracy_score(data_test.loc[:, 0], predictions)
# print('accuracy before normalize: ', accuracy)

#Normalize data sets
scaler = StandardScaler()
data_train_scaled = scaler.fit_transform(data_train.loc[:, 1:])
data_test_scaled = scaler.transform(data_test.loc[:, 1:])

y_train_scaled = data_train.loc[:, 0]
y_test_scaled = data_test.loc[:, 0]

clf.fit(data_train_scaled, y_train_scaled)
predictions = clf.predict(data_test_scaled)
accuracy_after = accuracy_score(y_test_scaled, predictions)
print(accuracy_after - accuracy_before)
print(accuracy_after)
# print(data_test)