/*
 * © Copyright 1996-2012 ECMWF.
 *
 * This software is licensed under the terms of the Apache Licence Version 2.0
 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
 * In applying this licence, ECMWF does not waive the privileges and immunities
 * granted to it by virtue of its status as an intergovernmental organisation nor
 * does it submit to any jurisdiction.
 */

#include "mars.h"

/* Checks that all single values contained in request 'a'
   are in request 'b' */
static err fix_check(const request* grib, const request* req, int cnt) {
    /* Should have everything which is not used for field ordering */

    static struct {
        const char* name; /* Name of the MARS parameter to check */
        boolean always;   /* If consistency grib<->request must always exist */
    } checks[] = {

        {
            "CLASS",
            true,
        },
        {
            "TYPE",
            true,
        },
        {
            "STREAM",
            true,
        },
        {
            "LEVTYPE",
            true,
        },
        {
            "_GRID_EW",
            false,
        },
        {
            "_GRID_NS",
            false,
        },
        {
            "ORIGIN",
            true,
        },

        {
            "PRODUCT",
            true,
        },
        {
            "SECTION",
            true,
        },
        {
            "METHOD",
            true,
        },
        {
            "SYSTEM",
            true,
        },

        /* Already in field_order */
        /* "EXPVER", */
        /* "DATE", */
        /* "TIME", */
        /* "REFERENCE", */
        /* "STEP", */
        /* "DOMAIN", */
        /* "DIAGNOSTIC", */
        /* "ITERATION", */
        /* "NUMBER", */
        /* "LEVELIST", */
        /* "PARAM", */
        /* "FREQUENCY", */
        /* "DIRECTION", */

    };

    int j = 0;
    if (mars.pseudogrib)
        return NOERR;

    if (grib && req) {
        while (j < NUMBER(checks)) {
            int i         = 0;
            const char* p = checks[j].name;
            const char* s = get_value(grib, p, i);
            while (s) {
                if (!value_exist(req, p, s)) {
                    const char* str  = get_value(grib, "STREAM", 0);
                    const char* clss = get_value(grib, "CLASS", 0);
                    boolean grid     = (strcmp(p, "_GRID_EW") == 0 || strcmp(p, "_GRID_NS") == 0);
                    /* In ERA 40 they do not care about the grid */
                    boolean supd = (strcmp(str, "SD") == 0 && strcmp(str, "E4") == 0);

                    if (supd || (!supd && !grid)) {
                        marslog(LOG_EROR, "Grib %d contains %s = %s. Grib description follows:", cnt, p, s);
                        print_all_requests(grib);
                        return -6;
                    }
                }
                s = get_value(grib, p, ++i);
            }
            j++;
        }
    }
    return NOERR;
}

static long compute_total(request* r) {
    long n       = 1;
    parameter* p = r->params;
    while (p) {
        n *= count_values(r, p->name);
        p = p->next;
    }
    return n;
}

static const char* fast_value(const request* r, const char* name) {
    if (r && name) {
        parameter* p = r->params;
        while (p) {
            if (p->name == name)
                return p->values ? p->values->name : NULL;
            p = p->next;
        }
    }
    return NULL;
}

static int cmp_no_hidden(const request* a, const request* b) {
    int n;

    static const char* repres = 0;

    if (repres == 0)
        repres = strcache("REPRES");

    if (a && b) {
        parameter* p = a->params;

        while (p) {
            const char* s = fast_value(a, p->name);
            const char* t = fast_value(b, p->name);

            if (s && t && (*p->name != '_') && p->name != repres) {

                n = (s != t);
                if (n != 0 && is_number(s))
                    n = atof(s) - atof(t);
                if (n) {
                    if (mars.debug)
                        marslog(LOG_DBUG, "Compare failed: %s -> %s <> %s", p->name, s, t);
                    return n;
                }
            }

            p = p->next;
        }
    }
    return 0;
}

static int find_order(request** v, request* g) {
    request* w = 0;
    request* u = *v;

    while (u) {
        if (cmp_no_hidden(u, g) == 0) {
            int order = u->order;
            if (w != 0)
                w->next = u->next;
            else
                *v = u->next;

#if 1
            /* try to reorganise the list so next time we scan the minumum */

            if (w != 0) {
                request* z = w->next;
                request* p = 0;

                while (z) {
                    p = z;
                    z = z->next;
                }


                if (p != 0) {
                    p->next = *v;
                    *v      = w->next;
                    w->next = 0;
                }
            }
#endif

            u->next = NULL;
            free_one_request(u);

            return order;
        }
        w = u;
        u = u->next;
    }

    return -1;
}

int find_long(long* p, int count, long search) {
    int i;
    for (i = 0; i < count; i++)
        if (p[i] == search)
            return TRUE;
    return FALSE;
}

static err init_subtype_tables(request* r,
                               char* subtype_group,
                               char* known_subtypes) {
    memset((void*)subtype_group, 0, 256);
    memset((void*)known_subtypes, 0, 256);

    if (count_values(r, "OBSGROUP") == 1) {
        request* dummy_request;
        request* expanded_request;
        int count;
        int i;

        /*----------------------------------------------------------------*/
        /* We expand a dummy request with the obstype value to get all    */
        /* the subtypes in the obstypes. We put 2 into every entry in the */
        /* table                                                          */
        /*----------------------------------------------------------------*/
        count = count_values(r, "OBSTYPE");
        for (i = 0; i < count; i++) {
            long st = atol(get_value(r, "OBSTYPE", i));
            if (st < 0 || st > 255) {
                marslog(LOG_EROR,
                        "Subtype %d is greater than 255. "
                        "If this is a valid subtype, please contact the MARS group",
                        st);
                return -6;
            }
            subtype_group[st] = 2;
        }

        /*----------------------------------------------------------------*/
        /* We expand a dummy request with the obsgroup value to get all   */
        /* the subtypes in the obsgroup. Whenever an entry contains 2     */
        /* it means that it was in the OBSTYPE, we put 1 in it meaning    */
        /* it is also in the obsgroup.                                    */
        /*----------------------------------------------------------------*/
        dummy_request = empty_request("ARCHIVE");
        set_value(dummy_request, "TYPE", "OB");
        set_value(dummy_request, "OBSTYPE", get_value(r, "OBSGROUP", 0));
        expanded_request = expand_mars_request(dummy_request);
        count            = count_values(expanded_request, "OBSTYPE");

        for (i = 0; i < count; i++) {
            long st = atol(get_value(expanded_request, "OBSTYPE", i));
            if (st < 0 || st > 255) {
                marslog(LOG_EROR,
                        "Subtype %d is greater than 255. "
                        "If this is a valid subtype, please contact the MARS group",
                        st);
                return -6;
            }
            if (subtype_group[st] == 2)
                subtype_group[st] = 1;
        }
        free_one_request(expanded_request);

        /*----------------------------------------------------------------*/
        /* After this, if there are any 2s left in the table, then        */
        /* OBSTYPE is not a subset of OBSGROUP.                           */
        /*----------------------------------------------------------------*/
        for (i = 0; i < 256; i++) {
            if (subtype_group[i] == 2) {
                marslog(LOG_EROR, "OBSTYPE %d is not a subset of OBSGROUP", i);
                marslog(LOG_EROR, "Add it to OBSGROUP %s in mars.def", get_value(r, "OBSGROUP", 0));
                return -6;
            }
        }

        /*----------------------------------------------------------------*/
        /* We expand a dummy request with the all the known subtypes      */
        /* (KNOWN).														  */
        /*----------------------------------------------------------------*/
        set_value(dummy_request, "OBSTYPE", "KNOWN");
        expanded_request = expand_mars_request(dummy_request);
        count            = count_values(expanded_request, "OBSTYPE");
        for (i = 0; i < count; i++) {
            long st = atol(get_value(expanded_request, "OBSTYPE", i));
            if (st < 0 || st > 255) {
                marslog(LOG_EROR,
                        "Subtype %d is greater than 255. "
                        "If you think this is a valid subtype, please contact the MARS team",
                        st);
                return -6;
            }
            known_subtypes[st] = TRUE;
        }

        free_one_request(dummy_request);
        free_one_request(expanded_request);
    }
    else {
        marslog(LOG_EROR, "Archiving requires only one OBSGROUP value");
        return -6;
    }

    return 0;
}

/* assumes only one date, one time, one range */
static void get_archive_period(request* r, time_interval* t) {
    long time      = atol(get_value(r, "TIME", 0));
    datetime begin = date_time_2_datetime(
        mars_date_to_julian(atol(get_value(r, "DATE", 0))),
        time / 100,
        time % 100,
        0);
    long range = atol(get_value(r, "RANGE", 0));

    /* range is in minutes */
    init_time_interval(t, begin, range * 60 + 59);
}

static err dhs_fb_check(dhsdata* dhs, request* original_req) {
    marslog(LOG_INFO, "No checking is carried out on extraction/feedback data");

    return 0;
}

static err dhs_obs_check(dhsdata* dhs, request* original_req) {
    long length = 65536;
    char buffer[65536];
    int cnt = 0;
    err e   = 0;
    FILE* f;
    char buf[80];
    int q = 0;
    const char* s;
    time_interval archive_period;
    long err_count = 0;

    long64 file_length = 0;

    /*----------------------------------------------------------------*/
    /* We want to make sure that data included in the file to be	  */
    /* archived complies with the description given in the request.	  */
    /* 1. the subtypes found in the BUFR files have to be known in    */
    /*    mars language. Otherwise failure.                           */
    /* 2. the subtypes in the OBSTYPE parameter have to be the same   */
    /*    as in the OBSGROUP or a subset of it. Otherwise failure.	  */
    /*	  This ensures that what is archived is what people who are   */
    /*    archiving think is being archived.                          */
    /* 3. There sould only be one date in the request.				  */
    /* 4. There sould only be one time in the request.				  */
    /* 5. There sould only be one range in the request.				  */
    /* 6. Messages which are not in the [date.time,date.time+range]   */
    /*    are not accepted, failure.                                  */
    /* 7. I should have started with this: any problem in the BUFR    */
    /*    file (key missing, wrong length etc..) is a failure.        */
    /*                                                                */
    /* Any other idea?                                                */
    /*----------------------------------------------------------------*/

    /*----------------------------------------------------------------*/
    /* We check subtypes assuming a subtype is less than 256 which is */
    /* currently the case. This speeds up dramatically data processing*/
    /* since we use a table.										  */
    /* If subtypes were codes using more than a byte in the future,   */
    /* this code should be changed (dream on).                        */
    /*                                                                */
    /* Ths subtype_group array contains all the subtypes in the       */
    /* OBSGROUP less the subtypes which are not in the OBSTYPE.       */
    /* Any discrepancy between those cause init_subtype_tables to     */
    /* return an error.                                               */
    /*----------------------------------------------------------------*/
    char subtype_group[256];
    char known_subtypes[256];

    e = init_subtype_tables(dhs->req, subtype_group, known_subtypes);
    if (e != 0)
        return e;

    if (count_values(dhs->req, "DATE") > 1) {
        marslog(LOG_EROR, "Only one date must be specified for archiving data");
        return -6;
    }
    if (count_values(dhs->req, "TIME") > 1) {
        marslog(LOG_EROR, "Only one time must be specified for archiving data");
        return -6;
    }

    get_archive_period(dhs->req, &archive_period);

    dhs->total_tosend = 0;

    start_timer();

    while (e == 0 && (s = get_value(dhs->req, "SOURCE", q++)) != NULL) {
        s = no_quotes(s);

        marslog(LOG_INFO, "Scanning and analysing %s", s);

        f = fopen(s, "r");
        if (f == NULL) {
            marslog(LOG_EROR | LOG_PERR, "ARCHIVE : '%s'", s);
            e = -6;
            break;
        }

        if (fseek(f, 0, SEEK_END) != 0) {
            marslog(LOG_EROR | LOG_PERR, "fseek(%s)", s);
            e = -2;
        }

        file_length = ftell(f);
        if (dhs->total_tosend == 0)
            set_value_long64(original_req, "_length", file_length);
        else
            add_value_long64(original_req, "_length", file_length);
        dhs->total_tosend += file_length;

        if (fseek(f, 0, SEEK_SET) != 0) {
            marslog(LOG_EROR | LOG_PERR, "fseek(%s)", s);
            e = -2;
        }

        length = sizeof(buffer);
        while (!e && (e = _readany(f, buffer, &length)) == NOERR || (e == BUF_TO_SMALL)) {
            packed_key k;
            packed_key* keyptr = &k;
            int keyOk;

            keyOk = get_packed_key(buffer, &k);
            if (keyOk)
                keyOk = verify_bufr_key(buffer, length, &k);
            if (keyOk) {
                datetime obs_date;
                long subtype;
                const char* obsgroup = get_value(dhs->req, "OBSGROUP", 0);
                /*------------------------------------------------------*/
                /* if conventional data, IDENT is checked.				*/
                /* since subtypes 82 to 88 are in the conventional group*/
                /* and these are satellite data, we skip them.			*/
                /*------------------------------------------------------*/
#if 0
				if (EQ(obsgroup,"CONVENTIONAL"))
				{
					if (k.header.subtype < 82 || k.header.subtype > 88)
						if (KEY_IDENT(&k)[0] == ' ')
						{
							 marslog(LOG_EROR,
								"IDENT of report %d, subtype %d is left justified.",
								cnt+1,k.header.subtype);
							 err_count++;
						}
				}
#endif
                /*------------------------------------------------------*/
                /* check subtypes to be archived						*/
                /*------------------------------------------------------*/
                subtype = k.header.subtype;
                if (subtype < 0 || subtype > 255) {
                    marslog(LOG_EROR,
                            "Subtype %d is greater than 255. "
                            "If this is a valid subtype, please contact the MARS team",
                            subtype);
                    err_count++;
                }
                else if (!known_subtypes[subtype]) {
                    marslog(LOG_EROR,
                            "Found report %d with subtype %d which "
                            "is unknown",
                            cnt + 1, subtype);
                    err_count++;
                    marslog(LOG_EROR, "If valid, add %d to the list of KNOWN subtypes in mars.def", subtype);
                }
                else if (!subtype_group[subtype]) {
                    marslog(LOG_EROR,
                            "Found report %d with subtype %d which "
                            "does not match the specified OBSTYPE/OBSGROUP",
                            cnt + 1,
                            subtype);
                    err_count++;
                }

                /*------------------------------------------------------*/
                /* check the message is in archive period				*/
                /*------------------------------------------------------*/
                obs_date = key_2_datetime(keyptr);
                if (obs_date < archive_period.begin || obs_date > archive_period.end && e == 0) {
                    char date[20];
                    char time[20];
                    print_key_date(keyptr, date);
                    print_key_time(keyptr, time);
                    marslog(LOG_EROR, "Bufr message n.%d %s %s is out of the archiving period specified in the request",
                            cnt + 1, date, time);
                    err_count++;
                }

                cnt++;
            }
            else {
                marslog(LOG_EROR,
                        "Key of report %d is not valid", cnt + 1);
                err_count++;
            }
            length = sizeof(buffer);

            if (err_count > 50)
                e = -6;
        }

        if (e == EOF)
            e = 0;
        if (f)
            fclose(f);

    } /* Loop on sources */

    if (err_count > 0)
        e = -6;

    if (e == NOERR) {
        marslog(LOG_INFO, "Archiving %d report(s) from %d file(s)", cnt, q - 1);
        dhs->count = cnt;
    }

    set_value(original_req, "_SIZE", "%d", cnt);

    stop_timer(buf);
    if (*buf)
        marslog(LOG_INFO, "Analyse time: %s", buf);

    return e;
}

typedef struct multidatabase {
    boolean done;
    long64 total_tosend;
    int databases;
} multidatabase;

static err sanity_check(request* r) {
    const char* suite = getenv("SUITE");
    const char* t     = get_value(r, "STREAM", 0);

    if (suite && EQ(suite, "bc") && t) {
        if ((!EQ(t, "SCWV") && !EQ(t, "SCDA"))) {
            int exp = atol(get_value(r, "EXPVER", 0));
            FILE* f = mail_open(mars.dhsmail, "sanity check failed stream: %s", t);
            mail_msg(f, "Archiving from suite bc stream %s", t);
            mail_request(f, "MARS Request:", r);
            mail_close(f);

            if (exp == 1) {
                marslog(LOG_EROR, "Archiving from suite %s stream %s. Exiting...", suite, t);
                return -10;
            }

            marslog(LOG_WARN, "Archiving from suite %s stream %s", suite, t);
        }
    }
    return NOERR;
}

static err dhs_grib_check(dhsdata* dhs, request* original_req) {
    long length = 32768;
    char buffer[32768];
    int cnt = 0;
    err e   = 0;
    FILE* f;
    char buf[80];
    int q = 0;
    const char *s, *t;
    int count, total;
    char* found;
    boolean isfg              = false;
    static multidatabase once = {false, 0, 0};
    request* u                = unwind_one_request(dhs->req);
    int dbcount               = count_values(dhs->req, "DATABASE");

    timer* read_for_analyse = get_timer("Read GRIBs for analyse", NULL, true);

    s = get_value(dhs->req, "TYPE", 0);
    t = get_value(dhs->req, "STREAM", 0);

    isfg = (s && t && EQ(s, "FG") && !EQ(t, "WAVE") && !EQ(t, "SCWV"));

    if (isfg) {
        request* k = un_first_guess_all(u);
        free_all_requests(u);
        u = k;
        marslog(LOG_INFO, "Data is first-guess");
        if (mars.debug)
            print_one_request(u);
        set_value(dhs->req, "TYPE", "FC");
    }


    /* If language accept multi TYPE/CLASS/..., update count_fields
       and fixed_check */

    count = count_fields(dhs->req);
    total = compute_total(dhs->req);

    /* Avoid performing the check for the same database */
    if (once.done) {
        marslog(LOG_INFO, "Source data already analysed");


        dhs->total_tosend = once.total_tosend;
        once.databases++;

        /* Reset it for further requests */
        if (once.databases == dbcount) {
            once.done         = false;
            once.databases    = 0;
            once.total_tosend = 0;
        }

        return NOERR;
    }

    found = NEW_ARRAY_CLEAR(char, total);

    dhs->total_tosend = 0;

    if (mars.debug)
        print_all_requests(u);

    start_timer();

    while (e == 0 && (s = get_value(dhs->req, "SOURCE", q++)) != NULL) {

        s = no_quotes(s);

        marslog(LOG_INFO, "Scanning and analysing %s", s);

        f = fopen(s, "r");
        if (f == NULL) {
            marslog(LOG_EROR | LOG_PERR, "ARCHIVE : '%s'", s);
            e = -6;
            break;
        }

        if (fseek(f, 0, SEEK_END) != 0) {
            marslog(LOG_EROR | LOG_PERR, "fseek(%s)", s);
            e = -2;
        }

        dhs->total_tosend += ftell(f);

        if (fseek(f, 0, SEEK_SET) != 0) {
            marslog(LOG_EROR | LOG_PERR, "fseek(%s)", s);
            e = -2;
        }


        length = sizeof(buffer);
        while (!e && (e = timed_readany(f, buffer, &length, read_for_analyse)) == NOERR || (e == BUF_TO_SMALL)) {
            int n      = -1;
            request* g = empty_request("GRIB");

            if ((e = grib_to_request(g, buffer, length)) != NOERR) {
                marslog(LOG_EROR, "Error while building MARS request from GRIB %d", cnt + 1);
                marslog(LOG_EROR, "MARS description for Grib %d is:", cnt + 1);
                print_all_requests(g);
            }

            if (mars.debug) {
                marslog(LOG_DBUG, "Field %d:", cnt);
                print_all_requests(g);
            }

            if ((e == NOERR) && (e = fix_check(g, dhs->req, cnt + 1)) == NOERR) {
                n = find_order(&u, g);

                if (n < 0) {
                    if (field_order(dhs->req, g) >= 0)
                        marslog(LOG_EROR, "Grib %d in '%s' is duplicated", cnt + 1, s);
                    else
                        marslog(LOG_EROR, "Grib %d is not in ARCHIVE request", cnt + 1);
                    marslog(LOG_EROR, "MARS description for Grib %d is:", cnt + 1);
                    print_all_requests(g);
                    e = -6;
                }
                else if (n >= total) {
                    marslog(LOG_EROR, "MARS internal error analysing grib %d", cnt + 1);
                    marslog(LOG_EROR, "MARS description for Grib %d is:", cnt + 1);
                    print_all_requests(g);
                    e = -6;
                }
                else if (found[n]) {
                    marslog(LOG_EROR, "Grib %d in '%s' is duplicated", cnt + 1, s);
                    marslog(LOG_EROR, "MARS description for Grib %d is:", cnt + 1);
                    print_all_requests(g);
                    e = -6;
                }
            }

            if ((e == NOERR) || (e == BUF_TO_SMALL)) {
                found[n] = true;
                cnt++;
            }

            length = sizeof(buffer);
            free_all_requests(g);
        }

        if (e == EOF)
            e = 0;
        if (f)
            fclose(f);

    } /* Loop on sources */

    free_all_requests(u);
    FREE(found);

#ifdef fujitsu
    free(buffer);
#endif

    if (!e && cnt != count) {
        marslog(LOG_EROR, "Wrong number of fields :");
        marslog(LOG_EROR, "Request describes %d field(s)", count);
        marslog(LOG_EROR, "Source contains %d field(s)", cnt);
        e = -6;
    }
    else if (e == NOERR) {
        if (dbcount > 1) {
            once.done         = true;
            once.total_tosend = dhs->total_tosend;
            once.databases++;
        }
        marslog(LOG_INFO, "Archiving %d field(s) from %d file(s)", cnt, q - 1);
        dhs->count = cnt;
    }

    set_value(original_req, "_SIZE", "%d", cnt);


    stop_timer(buf);
    if (*buf)
        marslog(LOG_INFO, "Analyse time: %s", buf);

    if (isfg)
        set_value(dhs->req, "TYPE", "FG"); /* Reset type */

    return e;
}

static err dhs_image_check(dhsdata* dhs) {
    err e = 0;
    FILE* f;
    int q = 0;
    const char* s;
    char buf[80];


    dhs->total_tosend = 0;
    start_timer();

    while (e == 0 && (s = get_value(dhs->req, "SOURCE", q++)) != NULL) {
        s = no_quotes(s);
        marslog(LOG_INFO, "Scanning %s", s);

        f = fopen(s, "r");
        if (f == NULL) {
            marslog(LOG_EROR | LOG_PERR, "ARCHIVE : '%s'", s);
            e = -6;
            break;
        }

        if (fseek(f, 0, SEEK_END) != 0) {
            marslog(LOG_EROR | LOG_PERR, "fseek(%s)", s);
            e = -2;
        }

        dhs->total_tosend += ftell(f);

        fclose(f);

    } /* Loop on sources */


    stop_timer(buf);
    if (*buf)
        marslog(LOG_INFO, "Scanning time: %s", buf);

    return e;
}

static err dhs_store_check(dhsdata* dhs) {
    long length = 32768;
    char buffer[32768];
    err e = 0;
    FILE* f;
    char buf[80];
    int q = 0;
    const char* s;

    dhs->total_tosend = 0;

    start_timer();

    while (e == 0 && (s = get_value(dhs->req, "SOURCE", q++)) != NULL) {

        s = no_quotes(s);

        marslog(LOG_INFO, "Scanning and analysing %s", s);

        f = fopen(s, "r");
        if (f == NULL) {
            marslog(LOG_EROR | LOG_PERR, "STORE : '%s'", s);
            e = -6;
            break;
        }

        if (fseek(f, 0, SEEK_END) != 0) {
            marslog(LOG_EROR | LOG_PERR, "fseek(%s)", s);
            e = -2;
        }

        dhs->total_tosend += ftell(f);

        rewind(f);

        /*---------------------------*/

        /* Check for GRIB or BUFR */

        if (_readany(f, buffer, &length) != -1) {
            marslog(LOG_EROR, "STORE: %s contains GRIB or BUFR", s);
            e = -7;
        }

        rewind(f);

        /* check for something else....*/

        /*---------------------------*/

        fclose(f);

    } /* Loop on sources */

    stop_timer(buf);
    if (*buf)
        marslog(LOG_INFO, "Analyse time: %s", buf);

    return e;
}


source* new_source(request* r) {
    source* s = NEW_CLEAR(source);
    s->cube   = new_hypercube(r);
}


void free_source(source* s) {
    free_hypercube(s->cube);
}

err source_open(source*) {
}

err source_read(source*) {
}
