numpy.core._exceptions._ArrayMemoryError: Unable to allocate 456. TiB for an array with shape (64834560, 967680) and data type float64

[pradeepdev-1995]

I tried to mitigate bias using OptimPreproc in my dataset. Using the code below

from aif360.algorithms.preprocessing.optim_preproc import OptimPreproc
from aif360.algorithms.preprocessing.optim_preproc_helpers.opt_tools import OptTools
from aif360.algorithms.preprocessing.optim_preproc_helpers.distortion_functions import get_distortion_adult
from aif360.metrics import ClassificationMetric
from aif360.metrics import BinaryLabelDatasetMetric
from aif360.datasets.multiclass_label_dataset import MulticlassLabelDataset
import pandas as pd
import numpy as np
data = pd.read_csv("Train.csv")
dataset = MulticlassLabelDataset(
                                 favorable_label=[0,3,1],
                                 unfavorable_label = [2],
                                 df=data,
                                 label_names=['Segmentation'],
                                 protected_attribute_names=['Age']
                                )

attr = dataset.protected_attribute_names[0]
idx = dataset.protected_attribute_names.index(attr)
privileged_groups = [{attr: dataset.privileged_protected_attributes[idx][0]}]
unprivileged_groups = [{attr: dataset.unprivileged_protected_attributes[idx][0]}]
metric_pred = BinaryLabelDatasetMetric(dataset,
                                       unprivileged_groups=unprivileged_groups,
                                       privileged_groups=privileged_groups)

def get_distortion_custom(vold, vnew):
    def adjustAge(a):
        if a == '>=70':
            return 70.0
        else:
            return float(a)

    bad_val = 3.0
    aOld = adjustAge(vold['Age'])
    aNew = adjustAge(vnew['Age'])

    # Age cannot be increased or decreased in more than a decade
    if np.abs(aOld-aNew) > 10.0:
        return bad_val

    # Penalty of 2 if age is decreased or increased
    if np.abs(aOld-aNew) > 0:
        return 2.0

#optimized preprocessing bias mitigation
optim_options = {
    "distortion_fun": get_distortion_custom,
    "epsilon": 0.05,
    "clist": [0.99, 1.99, 2.99],
    "dlist": [.1, 0.05, 0]
}

OP = OptimPreproc(OptTools, optim_options)
OP = OP.fit(dataset)
dataset_transf_train = OP.transform(dataset, transform_Y=True)

But it return the error like this

numpy.core._exceptions._ArrayMemoryError: Unable to allocate 456. TiB for an array with shape (64834560, 967680) and data type float64

So how to put our own distortion_fun in OptimPreproc for our own dataset