Run in Google Colab

View on GitHub

Dimension Reduction Techniques¶

In this hands-on notebook, we will explore various dimension reduction techniques using Python. Dimension reduction is powerful way to visualize high-dimensional data. We will cover the following techniques: PCA, MDS, Isomap, and UMAP.

Let's first import some packages and load the COIL-20 dataset, which is a popular dataset for testing dimension reduction techniques. The COIL-20 dataset contains images of 20 different objects, each captured at 72 different angles.

In [2]:

import pandas as pd
import numpy as np
import sklearn
import matplotlib.pyplot as plt
import seaborn as sns
import umap
from pathlib import Path
import warnings

df = pd.read_feather("../../Datasets/coil-20-proc.feather")

import pandas as pd import numpy as np import sklearn import matplotlib.pyplot as plt import seaborn as sns import umap from pathlib import Path import warnings df = pd.read_feather("../../Datasets/coil-20-proc.feather")

You should find several features named p# in the dataset, which represent pixel values of the images.

In [3]:

df.head()

df.head()

Out[3]:

	p0	p1	p2	p3	p4	p5	p6	p7	p8	p9	p10	p11	p12	p13	p14	p15	p16	p17	p18	p19	p20	p21	p22	p23	p24	p25	p26	p27	p28	p29	p30	p31	p32	p33	p34	p35	p36	p37	p38	p39	...	p16347	p16348	p16349	p16350	p16351	p16352	p16353	p16354	p16355	p16356	p16357	p16358	p16359	p16360	p16361	p16362	p16363	p16364	p16365	p16366	p16367	p16368	p16369	p16370	p16371	p16372	p16373	p16374	p16375	p16376	p16377	p16378	p16379	p16380	p16381	p16382	p16383	Filename	Object	Image
0	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	...	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	obj1__0.png	1	0
1	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	...	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	5	obj1__1.png	1	1
2	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	...	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	obj1__2.png	1	2
3	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	...	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	4	obj1__3.png	1	3
4	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	...	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	3	obj1__4.png	1	4

5 rows × 16387 columns

Let's visualize the images first. Each image has 128x128 pixels, and we will reshape them to visualize.

In [4]:

# Let's take a look at the dataset
# dimensions of each entry are 128x128 grayscale images
# columns named p0, p1, ..., p16383 (128*128 = 16384 pixels)
pixelColumns = [f'p{i}' for i in range(16384)]

def get_image(row):
    """
    Get the image from a row of the dataframe
    :param row: a row of the dataframe
    :return: a 128x128 numpy array
    """
    # Get the pixel values from the row
    # Columns starting with p
    pixels = row[pixelColumns].values
    # Reshape to 128x128
    image = pixels.reshape(128, 128)
    return (image/255).tolist()

# Let's create a matrix of images to visualize the dataset
num_objects = df.Object.nunique()
num_images = min(10,df.Image.nunique()) # Limit to 10 images per object for visualization purposes
# Plot a grid of images
# no spacing between images, just to visualize the dataset
fig, axes = plt.subplots(num_objects, num_images, figsize=(num_images, num_objects)
                          , constrained_layout=True)
# Loop through each object and image to plot
# use objectIndex and imageIndex
for objectIndex, objectId in enumerate(df.Object.unique()):
    # Get all images for this object
    object_images = df[df.Object == objectId]
    for imageIndex in range(min(num_images, len(object_images))):
        # Get the row corresponding to the image
        row = object_images.iloc[imageIndex]
        # Get the image from the row
        image = get_image(row)
        # Plot the image in the corresponding subplot
        axes[objectIndex, imageIndex].imshow(image, cmap='gray')
        # Hide axes ticks and labels
        # add labels for the first column with the object ID
        if imageIndex == 0:
            # Label the first column with the object ID
            axes[objectIndex, imageIndex].set_ylabel(f'Object {objectId}', fontsize=10)
            axes[objectIndex, imageIndex].axis('on')  # Ensure the axis is on for the first column to show the label
            # but no ticks on the x or y axis
            axes[objectIndex, imageIndex].set_xticks([])
            axes[objectIndex, imageIndex].set_yticks([])
        else:
            axes[objectIndex, imageIndex].axis('off')
            axes[objectIndex, imageIndex].set_ylabel('')
plt.suptitle('Sample Images from the COIL-20 Dataset', fontsize=20)
plt.show()

# Let's take a look at the dataset # dimensions of each entry are 128x128 grayscale images # columns named p0, p1, ..., p16383 (128*128 = 16384 pixels) pixelColumns = [f'p{i}' for i in range(16384)] def get_image(row): """ Get the image from a row of the dataframe :param row: a row of the dataframe :return: a 128x128 numpy array """ # Get the pixel values from the row # Columns starting with p pixels = row[pixelColumns].values # Reshape to 128x128 image = pixels.reshape(128, 128) return (image/255).tolist() # Let's create a matrix of images to visualize the dataset num_objects = df.Object.nunique() num_images = min(10,df.Image.nunique()) # Limit to 10 images per object for visualization purposes # Plot a grid of images # no spacing between images, just to visualize the dataset fig, axes = plt.subplots(num_objects, num_images, figsize=(num_images, num_objects) , constrained_layout=True) # Loop through each object and image to plot # use objectIndex and imageIndex for objectIndex, objectId in enumerate(df.Object.unique()): # Get all images for this object object_images = df[df.Object == objectId] for imageIndex in range(min(num_images, len(object_images))): # Get the row corresponding to the image row = object_images.iloc[imageIndex] # Get the image from the row image = get_image(row) # Plot the image in the corresponding subplot axes[objectIndex, imageIndex].imshow(image, cmap='gray') # Hide axes ticks and labels # add labels for the first column with the object ID if imageIndex == 0: # Label the first column with the object ID axes[objectIndex, imageIndex].set_ylabel(f'Object {objectId}', fontsize=10) axes[objectIndex, imageIndex].axis('on') # Ensure the axis is on for the first column to show the label # but no ticks on the x or y axis axes[objectIndex, imageIndex].set_xticks([]) axes[objectIndex, imageIndex].set_yticks([]) else: axes[objectIndex, imageIndex].axis('off') axes[objectIndex, imageIndex].set_ylabel('') plt.suptitle('Sample Images from the COIL-20 Dataset', fontsize=20) plt.show()

Let's create a version of the data that is easier to input to the models, with just the numeric values for pixels.

Also let's create a version that is standardized (X_scaled).

In [5]:

from sklearn.preprocessing import StandardScaler
# standardize
scaler = StandardScaler()
X = df[pixelColumns].values  # Get the pixel values as a numpy array
X_scaled = scaler.fit_transform(X)

from sklearn.preprocessing import StandardScaler # standardize scaler = StandardScaler() X = df[pixelColumns].values # Get the pixel values as a numpy array X_scaled = scaler.fit_transform(X)

The first technique we will explore is PCA (Principal Component Analysis). PCA is a linear dimension reduction technique that projects the data onto the directions of maximum variance. You can test it with and without standardization.

In [6]:

# Apply PCA (standardize and reduce dimensionality)
from sklearn.decomposition import PCA

dimensions = 2

# PCA
pca = PCA(n_components=dimensions)
X_pca = pca.fit_transform(X_scaled)
# add back to df named PC1,PC2,PC3...
for i in range(dimensions):
    df[f"PC{i+1}"] = X_pca[:,i]

# visualize first 2 components
plt.figure(figsize=(10, 10))
# color by Object
cmap = "tab20"
cmapFunction = plt.get_cmap(cmap)
for i,objectName in enumerate(df["Object"].unique()):
    color = cmapFunction(i)
    groupData = df[df["Object"] == objectName]
    plt.scatter(groupData["PC1"], groupData["PC2"], label=objectName, color=color)
# add legend
plt.legend()
# calculate the explained variance ratio
explained_variance = pca.explained_variance_ratio_
plt.xlabel(f"PC1 ({explained_variance[0]:.2%})")
plt.ylabel(f"PC2 ({explained_variance[1]:.2%})")


plt.title("PCA")
plt.show()

# Apply PCA (standardize and reduce dimensionality) from sklearn.decomposition import PCA dimensions = 2 # PCA pca = PCA(n_components=dimensions) X_pca = pca.fit_transform(X_scaled) # add back to df named PC1,PC2,PC3... for i in range(dimensions): df[f"PC{i+1}"] = X_pca[:,i] # visualize first 2 components plt.figure(figsize=(10, 10)) # color by Object cmap = "tab20" cmapFunction = plt.get_cmap(cmap) for i,objectName in enumerate(df["Object"].unique()): color = cmapFunction(i) groupData = df[df["Object"] == objectName] plt.scatter(groupData["PC1"], groupData["PC2"], label=objectName, color=color) # add legend plt.legend() # calculate the explained variance ratio explained_variance = pca.explained_variance_ratio_ plt.xlabel(f"PC1 ({explained_variance[0]:.2%})") plt.ylabel(f"PC2 ({explained_variance[1]:.2%})") plt.title("PCA") plt.show()

Now let's try to visualize the dataset using MDS (Multidimensional Scaling). MDS is a non-linear technique that tries to preserve the pairwise distances between points in the high-dimensional space when mapping to a lower-dimensional space. Try to use different dissimilarity metrics (e.g., euclidean, cosine) to see how it affects the results.

In [ ]:

# MDS
from sklearn.manifold import MDS

X_input = X_scaled

# You will see it takes a long time to run MDS on the full dataset.
# We can reduce the dimensionality first with PCA or use a smaller dataset for MDS.
# you can uncomment the following lines to reduce the dimensionality first with PCA before applying MDS.

# pcaDimensions = 50
# pca = PCA(n_components=pcaDimensions)
# X_input = pca.fit_transform(X_input)


mds = MDS(n_components=dimensions,
          dissimilarity='euclidean')
# Fit MDS to the pairwise data
X_mds = mds.fit_transform(X_input)
# add back to df named MDS1,MDS2,MDS3...
for i in range(dimensions):
    df[f"MDS{i+1}"] = X_mds[:,i]
# visualize first 2 components
plt.figure(figsize=(10, 10))
# color by Object
cmap = "tab20"
cmapFunction = plt.get_cmap(cmap)
for i,objectName in enumerate(df["Object"].unique()):
    color = cmapFunction(i)
    groupData = df[df["Object"] == objectName]
    plt.scatter(groupData["MDS1"], groupData["MDS2"], label=objectName, color=color)
# add legend
plt.legend()
plt.xlabel("MDS1")
plt.ylabel("MDS2")
plt.title("MDS")
plt.show()

# MDS from sklearn.manifold import MDS X_input = X_scaled # You will see it takes a long time to run MDS on the full dataset. # We can reduce the dimensionality first with PCA or use a smaller dataset for MDS. # you can uncomment the following lines to reduce the dimensionality first with PCA before applying MDS. # pcaDimensions = 50 # pca = PCA(n_components=pcaDimensions) # X_input = pca.fit_transform(X_input) mds = MDS(n_components=dimensions, dissimilarity='euclidean') # Fit MDS to the pairwise data X_mds = mds.fit_transform(X_input) # add back to df named MDS1,MDS2,MDS3... for i in range(dimensions): df[f"MDS{i+1}"] = X_mds[:,i] # visualize first 2 components plt.figure(figsize=(10, 10)) # color by Object cmap = "tab20" cmapFunction = plt.get_cmap(cmap) for i,objectName in enumerate(df["Object"].unique()): color = cmapFunction(i) groupData = df[df["Object"] == objectName] plt.scatter(groupData["MDS1"], groupData["MDS2"], label=objectName, color=color) # add legend plt.legend() plt.xlabel("MDS1") plt.ylabel("MDS2") plt.title("MDS") plt.show()

The next technique we will explore is Isomap. Isomap is a non-linear dimension reduction technique that extends MDS by incorporating geodesic distances on a nearest-neighbor graph. This allows it to capture the underlying manifold structure of the data.

In [8]:

# Now try Isomap
from sklearn.manifold import Isomap

dimensions = 2

X_input = X_scaled

# Isomap
isomap = Isomap(n_neighbors=5,
                n_components=dimensions)

# disable warnings for this part:
with warnings.catch_warnings():
    warnings.simplefilter("ignore")  # ignore warnings from Isomap
    # fit and transform
    X_isomap = isomap.fit_transform(X_input)

# add back to df named Isomap1,Isomap2,Isomap3...
for i in range(dimensions):
    df[f"Isomap{i+1}"] = X_isomap[:,i]

# visualize first 2 components

plt.figure(figsize=(10, 10))
# color by Object

cmap = "tab20"

cmapFunction = plt.get_cmap(cmap)

for i,objectName in enumerate(df["Object"].unique()):
    color = cmapFunction(i)
    groupData = df[df["Object"] == objectName]
    plt.scatter(groupData["Isomap1"], groupData["Isomap2"], label=objectName, color=color)
# add legend
plt.legend()
plt.xlabel("Isomap1")
plt.ylabel("Isomap2")
plt.title("Isomap")
plt.show()

# Now try Isomap from sklearn.manifold import Isomap dimensions = 2 X_input = X_scaled # Isomap isomap = Isomap(n_neighbors=5, n_components=dimensions) # disable warnings for this part: with warnings.catch_warnings(): warnings.simplefilter("ignore") # ignore warnings from Isomap # fit and transform X_isomap = isomap.fit_transform(X_input) # add back to df named Isomap1,Isomap2,Isomap3... for i in range(dimensions): df[f"Isomap{i+1}"] = X_isomap[:,i] # visualize first 2 components plt.figure(figsize=(10, 10)) # color by Object cmap = "tab20" cmapFunction = plt.get_cmap(cmap) for i,objectName in enumerate(df["Object"].unique()): color = cmapFunction(i) groupData = df[df["Object"] == objectName] plt.scatter(groupData["Isomap1"], groupData["Isomap2"], label=objectName, color=color) # add legend plt.legend() plt.xlabel("Isomap1") plt.ylabel("Isomap2") plt.title("Isomap") plt.show()

Finally, we will explore UMAP (Uniform Manifold Approximation and Projection). UMAP is a non-linear dimension reduction technique that is particularly effective for visualizing complex data structures. It is based on manifold learning and can be thought of as a generalization of t-SNE.

In [9]:

# repeat same for umap
reducer = umap.UMAP(n_components=dimensions,
                    # metric="euclidean",
                    # n_epochs=1000,
                    # n_neighbors=15,
                    # min_dist=0.1,
                    # negative_sample_rate=5,
                    )
X_umap = reducer.fit_transform(X_scaled)

# add back to df named UMAP1,UMAP2,UMAP3...
for i in range(dimensions):
    df[f"UMAP{i+1}"] = X_umap[:,i]

# visualize first 2 components

plt.figure(figsize=(10, 10))
# color by Object
cmap = "tab20"

cmapFunction = plt.get_cmap(cmap)

for i,objectName in enumerate(df["Object"].unique()):
    color = cmapFunction(i)
    groupData = df[df["Object"] == objectName]
    plt.scatter(groupData["UMAP1"], groupData["UMAP2"], label=objectName, color=color)
    # place a label on each group centroid
    # calculate centroid
    centroid_x = groupData["UMAP1"].mean()
    centroid_y = groupData["UMAP2"].mean()
    # place label at centroid
    plt.text(centroid_x, centroid_y, objectName,
             fontsize=9,
             ha='center', va='center',
             color='black', alpha=1.0,
             zorder=5, # to ensure text is above the scatter points
    )
# add legend
plt.legend()
# Do not label UMAP axes.
# disable axes labels for UMAP, as they do not have a direct interpretation like PCA
plt.axis('off') 
plt.title("UMAP")

plt.show()

# repeat same for umap reducer = umap.UMAP(n_components=dimensions, # metric="euclidean", # n_epochs=1000, # n_neighbors=15, # min_dist=0.1, # negative_sample_rate=5, ) X_umap = reducer.fit_transform(X_scaled) # add back to df named UMAP1,UMAP2,UMAP3... for i in range(dimensions): df[f"UMAP{i+1}"] = X_umap[:,i] # visualize first 2 components plt.figure(figsize=(10, 10)) # color by Object cmap = "tab20" cmapFunction = plt.get_cmap(cmap) for i,objectName in enumerate(df["Object"].unique()): color = cmapFunction(i) groupData = df[df["Object"] == objectName] plt.scatter(groupData["UMAP1"], groupData["UMAP2"], label=objectName, color=color) # place a label on each group centroid # calculate centroid centroid_x = groupData["UMAP1"].mean() centroid_y = groupData["UMAP2"].mean() # place label at centroid plt.text(centroid_x, centroid_y, objectName, fontsize=9, ha='center', va='center', color='black', alpha=1.0, zorder=5, # to ensure text is above the scatter points ) # add legend plt.legend() # Do not label UMAP axes. # disable axes labels for UMAP, as they do not have a direct interpretation like PCA plt.axis('off') plt.title("UMAP") plt.show()

/Users/filsilva/miniforge3/envs/dataviz/lib/python3.11/site-packages/sklearn/utils/deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
  warnings.warn(
OMP: Info #276: omp_set_nested routine deprecated, please use omp_set_max_active_levels instead.