data_generation.py

import numpy as np
import pandas as pd
import os.path


# synthetic case from uds
def correlated_data(m, n, sigma, f):
    l = int(n / 2)
    Z = np.random.normal(0, 1, (m, l))
    A = np.matrix(np.random.uniform(0, 1, (l, l)))
    X1 = Z * A
    B = np.matrix(np.random.uniform(0, 0.5, (l, l)))
    W = X1 * B
    E = np.random.normal(0, sigma, (m, l))
    X2 = f(W) + E
    result = np.append(X1, X2, axis=1)
    print(result)
    return result


def generate_uncorrelated_data(m, n):
    return np.random.normal(0, 1, (m, n))


def func1(X):
    return 2 * X + 1


def func2(X):
    return np.log2(np.abs(X) + 1)


def synthetic_data_uni(m, r, s, sigma=0.1):
    r_dims = np.random.uniform(-0.5, 0.5, (m, r)) if r > 0 else np.empty((m, r))
    parity_dim = -(np.count_nonzero(r_dims > 0, axis=1) % 2 * 2 - 1).reshape(m, 1) * np.random.uniform(0, 0.5,
                                                                                                       (m,
                                                                                                        1)) if r > 0 else np.empty(
        (m, r))
    s_dims = np.random.uniform(-0.5, 0.5, (m, s))
    data = np.concatenate((r_dims, parity_dim, s_dims), axis=1)
    if sigma:
        e = np.random.normal(0, sigma, (m, r + s + 1))
        data = data + e

    return data


def synthetic_data_uni_negative(m, r, s, sigma=0.1):
    r_dims = np.random.uniform(-0.5, 0.5, (m, r)) if r > 0 else np.empty((m, r))
    parity_dim = (np.count_nonzero(r_dims > 0, axis=1) % 2 * 2 - 1).reshape(m, 1) * np.random.uniform(0, 0.5,
                                                                                                      (m,
                                                                                                       1)) if r > 0 else np.empty(
        (m, r))
    s_dims = np.random.uniform(-0.5, 0.5, (m, s))
    data = np.concatenate((r_dims, parity_dim, s_dims), axis=1)
    if sigma:
        e = np.random.normal(0, sigma, (m, r + s + 1))
        data = data + e

    return data


def synthetic_data_gauss(m, r, s, sigma=0.1):
    r_dims = np.random.normal(0, 1, (m, r)) if r > 0 else np.empty((m, r))
    parity_dim = -(np.count_nonzero(r_dims > 0, axis=1) % 2 * 2 - 1).reshape(m, 1) * np.abs(np.random.normal(0, 1,
                                                                                                             (m,
                                                                                                              1))) if r > 0 else np.empty(
        (m, r))
    s_dims = np.random.normal(0, 1, (m, s))
    data = np.concatenate((r_dims, parity_dim, s_dims), axis=1)
    if sigma:
        e = np.random.normal(0, sigma, (m, r + s + 1))
        data = data + e

    return data


def synthetic_with_nearcopies(m, k, l, sigma=0.1):
    k_dims = np.repeat(np.random.uniform(-0.5, 0, (m, 1)), k, axis=1) if k > 0 else np.empty((m, k))
    l_dims = np.repeat(np.random.uniform(0, 0.5, (m, 1)), l, axis=1) if l > 0 else np.empty((m, l))

    data = np.concatenate((k_dims, l_dims), axis=1)
    if sigma:
        e = np.random.normal(0, sigma, (m, k + l))
        data = data + e

    return data


def synthetic_cube_in_cube(m, r, i, side, sigma=0.5):
    if r < 1:
        raise ValueError
    h = int(m * sigma)
    range = [-0.5, 0] if side == 'l' else [-0.25, 0.25] if side == 'm' else [0, 0.5]
    contra_range = [0, 0.5] if side == 'l' else [-0.25, 0.25] if side == 'm' else [-0.5, 0]
    r_dims = np.concatenate((
        # np.concatenate(
        #     (np.random.uniform(range[0], range[1], (h, 1)),
        #      np.random.uniform(contra_range[0], contra_range[1], (h, 1))),
        #     axis=1)
        np.random.uniform(range[0], range[1], (h, r))
        , np.random.uniform(-0.5, 0.5, (m - h, r))), axis=0)
    i_dims = np.random.uniform(-0.5, 0.5, (m, i)) if i > 0 else np.empty((m, i))
    data = np.concatenate((r_dims, i_dims), axis=1)

    return data


def synthetic_cjs():
    return np.concatenate((np.concatenate((np.random.normal(0, 1, (100, 1)), np.random.normal(2, 1, (100, 1))), axis=1),
                           np.concatenate((np.random.normal(4, 1, (100, 1)), np.random.normal(5, 1, (100, 1))),
                                          axis=1)), axis=0)

# def blobs(rows):
#     blobs_number = 4
#     dims = 4
#     l = int(rows/blobs_number)
#     blob1 = np.random.normal(0, 1, (l, dims)) + np.concatenate((np.ones((l, 1)) * -3, np.ones((l, 1)) * -3, np.ones((l, 1)) * -3, np.ones((l, 1)) * -3), axis=1)
#     blob2 = np.random.normal(0, 1, (l, dims)) + np.concatenate((np.ones((l, 1)) * 0, np.ones((l, 1)) * 0, np.ones((l, 1)) * 0, np.ones((l, 1)) * 0), axis=1)
#     blob3 = np.random.normal(0, 1, (l, dims)) + np.concatenate((np.ones((l, 1)) * 3, np.ones((l, 1)) * 3, np.ones((l, 1)) * 3, np.ones((l, 1)) * 3), axis=1)
#     blob4 = np.random.normal(0, 1, (l, dims)) + np.concatenate((np.ones((l, 1)) * 6, np.ones((l, 1)) * 6, np.ones((l, 1)) * 6, np.ones((l, 1)) * 6), axis=1)
#
#     return np.concatenate((blob1, blob2, blob3, blob4), axis=0)
#     # return np.concatenate((blob1, blob2, blob3), axis=0)

def cubes(rows):
    cubes_number = 4
    dims = 4
    l = int(rows/cubes_number)
    blob1 = np.random.uniform(0, 1, (l, dims)) + np.concatenate((np.ones((l, 1)) * -1.7, np.ones((l, 1)) * -1.7, np.ones((l, 1)) * -1.7, np.ones((l, 1)) * -1.7), axis=1)
    blob2 = np.random.uniform(0, 1, (l, dims)) + np.concatenate((np.ones((l, 1)) * 0, np.ones((l, 1)) * 0, np.ones((l, 1)) * 0, np.ones((l, 1)) * 0), axis=1)
    blob3 = np.random.uniform(0, 1, (l, dims)) + np.concatenate((np.ones((l, 1)) * 1.5, np.ones((l, 1)) * 1.5, np.ones((l, 1)) * 1.5, np.ones((l, 1)) * 1.5), axis=1)
    blob4 = np.random.uniform(0, 1, (l, dims)) + np.concatenate((np.ones((l, 1)) * 3, np.ones((l, 1)) * 3, np.ones((l, 1)) * 3, np.ones((l, 1)) * 3), axis=1)
    background = np.random.uniform(-2, 4, (l, dims))
    # blob3 = np.random.normal(0, 1, (l, dims)) + np.concatenate((np.ones((l, 1)) * 3, np.ones((l, 1)) * 3, np.ones((l, 1)) * 3, np.ones((l, 1)) * 3), axis=1)
    # blob4 = np.random.normal(0, 1, (l, dims)) + np.concatenate((np.ones((l, 1)) * 6, np.ones((l, 1)) * 6, np.ones((l, 1)) * 6, np.ones((l, 1)) * 6), axis=1)

    # return np.concatenate((blob1, blob2, background), axis=0)
    return np.concatenate((blob1, blob2, blob3, blob4, background), axis=0)
    # return np.concatenate((blob1, blob2, blob3, background), axis=0)


def append_irrelevant_features(file, n):
    if n == 0:
        raise ValueError("# of irrelevant features is 0")
    data = pd.read_csv(file, delimiter=";", header=None, na_values='?')
    rows = data.shape[0]
    last_dim = data.shape[1] - 1
    irrel_data = np.random.uniform(-0.5, 0.5, (rows, n))
    return np.concatenate([data.loc[:, :last_dim - 1], irrel_data, data.loc[:, last_dim].to_frame()], axis=1) if (data[last_dim] == 0).all() \
        else np.concatenate([data, irrel_data], axis=1)


def generate():
    # -------generating dataset
    # data = synthetic_cube_in_cube(rows, rel_features, irrel_features, 'l')
    # data__ = synthetic_cjs()
    # data = correlated_data(rows, rel_features + irrel_features, 1, func1)
    # data = cubes(rows)
    # # add zeroes as default class
    # data = np.concatenate((data, np.zeros((data.shape[0], 1))), axis=1)
    # -------appending irrelevant features to existing dataset
    data = append_irrelevant_features(source, irrel_features)
    # storing to disk
    pd.DataFrame(data).to_csv(file, sep=';', header=False, index=False, float_format='%.2f')


if __name__ == '__main__':
    # ------APPENDING IRRELEVANT FEATURES
    for source_name in [
        "2d_2_cubes_aligned.csv",
        "2d_2_cubes_xor.csv",
        "2d_3_cubes_aligned_xor.csv",
        "3d_2_cubes_aligned.csv",
        "3d_2_cubes_xor.csv",
        "3d_3_cubes_aligned.csv",
        "3d_3_cubes_aligned_xor.csv",
        "3d_3_cubes_xor.csv",
        "3d_4_cubes_1_aligned_xor.csv",
        "3d_4_cubes_2_aligned.csv",
        "3d_4_cubes_xor.csv",
        "4d_2_cubes_aligned.csv",
        "4d_3_cubes_aligned_xor.csv",
        "4d_3_cubes_xor.csv",
        "4d_4_cubes_2_aligned.csv",
        "4d_4_cubes_aligned_xor.csv",
        "4d_4_cubes_xor.csv",
    ]:
        for i in [4,5,6,7,8,9,10]:
        #     file = 'synthetic_cases/synthetic_cube_in_cube_10.csv'
            source = 'synthetic_cases/cubes/' + source_name
            file = 'synthetic_cases/cubes/' + source_name.replace(".csv", "") + '_' + str(i) + '.csv'

            if os.path.isfile(file):
                raise ValueError(file + " already exists!")

            # parameters
            rows = 4000
            # rel_features = 10
            irrel_features = i

            generate()
    
    # ----GENERATION----
    # file = 'synthetic_cases/cubes/4d_4_cubes_xor.csv'
    #
    # if os.path.isfile(file):
    #     raise ValueError(file + " already exists!")
    #
    # # parameters
    # rows = 4000
    # rel_features = 2
    # irrel_features = 0
    #
    # generate()