Press "Enter" to skip to content

)

.

ぃ灵彧が的学习日志

.

.

.

【深度学习前沿应用】图像风格迁移

(一)、数据加载及预处理
(三)、定义损失函数与优化过程

## (一)、数据加载及预处理

```import paddle
import paddle as P
from paddle import ParamAttr
import paddle.nn as nn
import paddle.nn.functional as F
from paddle.nn import Conv2D, BatchNorm, Linear, Dropout, AdaptiveAvgPool2D, MaxPool2D, AvgPool2D
import numpy as np
import PIL
from sklearn.neighbors import KNeighborsRegressor
import os
from skimage import io
from skimage.color import rgb2lab, lab2rgb
from skimage.transform import resize
import matplotlib.pyplot as plt
import math
place = P.CUDAPlace(0)
P.disable_static(place)
image_style = '/home/aistudio/work/风格图像/'
print('风格图像：')
plt.imshow(io.imread(image_style+'虚空夜月.jpg'))
plt.show()
image_content = '/home/aistudio/work/浮光楼阁.png'
print('内容图像：')
plt.imshow(io.imread(image_content))
plt.show()
edge_size = 1
k = 4```

## (二)、创建数据集

```L_train = []
ab_train = []
for file in os.listdir(image_style):
L0, ab0, _, _ = create_dataset(image_style+file)
L_train.extend(L0)
ab_train.extend(ab0)
knnr = KNeighborsRegressor(n_neighbors=k, weights='distance')
knnr.fit(L_train, ab_train)```

## (三)、模型训练

```L_train = []
ab_train = []
for file in os.listdir(image_style):
L0, ab0, _, _ = create_dataset(image_style+file)
L_train.extend(L0)
ab_train.extend(ab0)
knnr = KNeighborsRegressor(n_neighbors=k, weights='distance')
knnr.fit(L_train, ab_train)```

## (四)、绘制图像

```def rebuild(image_content):
L, _, row, col = create_dataset(image_content)
ab = knnr.predict(L).reshape([row-2*edge_size, col-2*edge_size, -1])
Lab = np.zeros([row, col, 3])
Lab[:,:,0] = rgb2lab(io.imread(image_content))[:,:,0]
for x in range(edge_size, row-edge_size):
for y in range(edge_size, col-edge_size):
Lab[x, y, 1] = ab[x-edge_size, y-edge_size, 0]
Lab[x, y, 2] = ab[x-edge_size, y-edge_size, 1]
return Lab```

## (一)、数据加载及预处理

```import paddle
import paddle as P
from paddle import ParamAttr
import paddle.nn as nn
import paddle.nn.functional as F
from paddle.nn import Conv2D, BatchNorm, Linear, Dropout, AdaptiveAvgPool2D, MaxPool2D, AvgPool2D
import numpy as np
import PIL
from sklearn.neighbors import KNeighborsRegressor
import os
from skimage import io
from skimage.color import rgb2lab, lab2rgb
from skimage.transform import resize
import matplotlib.pyplot as plt
import math
# place = P.CUDAPlace(0)
# P.disable_static(place)
means = np.array([0.485, 0.456, 0.406])
image_style = io.imread('/home/aistudio/work/风格图像/虚空夜月.jpg')
image_style = resize(image_style, (384,512))
image_style = (image_style - means) * 255
image_content = io.imread('/home/aistudio/work/浮光楼阁.png')
sz = image_content.shape[:2]
image_content = resize(image_content, (384,512))
image_content = (image_content - means) * 255
image_transfer = 0.3*image_content + 0.7*np.random.randint(-20, 20, (image_content.shape[0],image_content.shape[1],image_content.shape[2]))
print('初始化的迁移图像：')
plt.imshow(PIL.Image.fromarray(np.uint8(image_transfer/255+means)))
plt.show()
image_transfer = P.to_tensor(image_transfer[:,:,:,None].transpose([3,2,0,1]).astype('float32'), stop_gradient=False)```

## (二)、模型配置

1. 卷积块定义：

```class ConvBlock(nn.Layer):
def __init__(self, input_channels, output_channels, groups, name=None):
super(ConvBlock, self).__init__()
self.groups = groups
self._conv_1 = Conv2D(
in_channels=input_channels,
out_channels=output_channels,
kernel_size=3,
stride=1,
padding=1,
weight_attr=ParamAttr(name=name + "1_weights"),
bias_attr=False)
if groups == 2 or groups == 3 or groups == 4:
self._conv_2 = Conv2D(
in_channels=output_channels,
out_channels=output_channels,
kernel_size=3,
stride=1,
padding=1,
weight_attr=ParamAttr(name=name + "2_weights"),
bias_attr=False)
if groups == 3 or groups == 4:
self._conv_3 = Conv2D(
in_channels=output_channels,
out_channels=output_channels,
kernel_size=3,
stride=1,
padding=1,
weight_attr=ParamAttr(name=name + "3_weights"),
bias_attr=False)
if groups == 4:
self._conv_4 = Conv2D(
in_channels=output_channels,
out_channels=output_channels,
kernel_size=3,
stride=1,
padding=1,
weight_attr=ParamAttr(name=name + "4_weights"),
bias_attr=False)
# self._pool = MaxPool2D(kernel_size=2, stride=2, padding=0)
self._pool = AvgPool2D(kernel_size=2, stride=2, padding=0)
def forward(self, inputs):
conv1 = self._conv_1(inputs)
x = F.relu(conv1)
if self.groups == 2 or self.groups == 3 or self.groups == 4:
conv2 = self._conv_2(x)
x = F.relu(conv2)
if self.groups == 3 or self.groups == 4:
x = self._conv_3(x)
x = F.relu(x)
if self.groups == 4:
x = self._conv_4(x)
x = F.relu(x)
x = self._pool(x)
return x, conv1, conv2```

1. 前向传播函数：

```def forward(self, inputs):
conv1 = self._conv_1(inputs)
x = F.relu(conv1)
if self.groups == 2 or self.groups == 3 or self.groups == 4:
conv2 = self._conv_2(x)
x = F.relu(conv2)
if self.groups == 3 or self.groups == 4:
x = self._conv_3(x)
x = F.relu(x)
if self.groups == 4:
x = self._conv_4(x)
x = F.relu(x)
x = self._pool(x)
return x, conv1, conv2```

1. VGG网络定义：

```class VGGNet(nn.Layer):
def __init__(self):
super(VGGNet, self).__init__()
self.groups = [2, 2, 4, 4, 4]
self._conv_block_1 = ConvBlock(3, 64, self.groups[0], name="conv1_")
self._conv_block_2 = ConvBlock(64, 128, self.groups[1], name="conv2_")
self._conv_block_3 = ConvBlock(128, 256, self.groups[2], name="conv3_")
self._conv_block_4 = ConvBlock(256, 512, self.groups[3], name="conv4_")
self._conv_block_5 = ConvBlock(512, 512, self.groups[4], name="conv5_")
def forward(self, inputs):
x, conv1_1, _ = self._conv_block_1(inputs)
x, conv2_1, _ = self._conv_block_2(x)
x, conv3_1, _ = self._conv_block_3(x)
x, conv4_1, conv4_2 = self._conv_block_4(x)
_, conv5_1, _ = self._conv_block_5(x)
return conv4_2, conv1_1, conv2_1, conv3_1, conv4_1, conv5_1```

1. 定义网络作特征提取器

```vgg19 = VGGNet()
vgg19.set_state_dict(P.load('/home/aistudio/work/vgg19_ww.pdparams'))
vgg19.eval()```

## (三)、定义损失函数与优化过程

1. 定义内容损失：

```def contentloss(content, transfer):
return 0.5 * P.sum((content - transfer)**2)```

1. 计算特征映射的Gram矩阵

```def gram(feature):
_, c, h, w = feature.shape
feature = feature.reshape([c,h*w])
return P.matmul(feature, feature.transpose([1,0]))```

1. 定义风格损失：

Gram Matrix实际上可看做是feature之间的偏心协方差矩阵（即没有减去均值的协方差矩阵），在feature map中，每一个数字都来自于一个特定滤波器在特定位置的卷积，因此每个数字就代表一个特征的强度，而Gram计算的实际上是两两特征之间的相关性，哪两个特征是同时出现的，哪两个是此消彼长的等等，同时，Gram的对角线元素，还体现了每个特征在图像中出现的量，因此，Gram有助于把握整个图像的大体风格。有了表示风格的Gram Matrix，要度量两个图像风格的差异，只需比较他们Gram Matrix的差异即可。

```def styleloss(style, transfer, weight):
loss = 0
for i in range(len(style)):
gram_style = gram(style[i])
gram_transfer = gram(transfer[i])
_, c, h, w = style[i].shape
loss += weight[i] * P.sum((gram_style - gram_transfer)**2) / (2*c*h*w)**2
return loss```

1. 自定义一个Adam优化器，更新迁移函数图像的参数

```def adam(image_transfer, m, v, g, t, η, β1=0.9, β2=0.999, ε=1e-8):
m = β1*m + (1-β1)*g
v = β2*v + (1-β2)*g**2
m_hat = m / (1 - β1**t)
v_hat = v / (1 - β2**t)
image_transfer -= η*m_hat / (P.sqrt(v_hat) + ε)
return image_transfer, m, v```

## (四)、生成图像

1. 单步迭代

```def trainer(image_transfer, m, v, net, features_content, features_style, t, η):
features_transfer = net(image_transfer)
loss_content = contentloss(features_content[0], features_transfer[0])
weight_style = [0.5,1.0,1.5,3.0,4.0]
loss_style = styleloss(features_style[1:], features_transfer[1:], weight_style)
loss = 1e0*loss_content + 1e3*loss_style
net.clear_gradients()
gradients = P.grad(loss, image_transfer)[0]
m,v=0,0
image_transfer, m, v = adam(image_transfer, m, v, gradients, t, η)
return image_transfer, m, v```

1. 多轮迭代

```def train(image_transfer, net, epoch_num):
features_content = net(P.to_tensor(image_content[:,:,:,None].transpose([3,2,0,1]).astype('float32')))
features_style = net(P.to_tensor(image_style[:,:,:,None].transpose([3,2,0,1]).astype('float32')))
m = P.zeros_like(image_transfer)
v = P.zeros_like(image_transfer)
for epoch in range(epoch_num):
image_transfer, m, v = trainer(image_transfer, m, v, net, features_content, features_style, epoch+1, 2)
if (epoch) % 50 == 0:
print('Epoch: ', epoch+1)
im = np.squeeze(image_transfer.numpy().transpose([2,3,1,0]))
im = im/255 + means
im = resize(im, sz)
im = PIL.Image.fromarray(np.uint8(im*255))
plt.imshow(im)
plt.show()```

1. 执行训练：

`train(image_transfer, vgg19, 250)`

## 三、基于PaddleHub的图像风格迁移

```import paddlehub as hub
import cv2
import matplotlib.image as mpimg
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
! hub install stylepro_artistic
stylepro_artistic = hub.Module(name="stylepro_artistic")
results = stylepro_artistic.style_transfer(images=[{

'content': cv2.imread("work/浮光楼阁.png"),
'styles': [cv2.imread("work/风格图像/虚空夜月.jpg")]}],
alpha = 1.0,
visualization = True)
# 原图展示
test_img_path = "work/浮光楼阁.png"
img = mpimg.imread(test_img_path)
plt.figure(figsize=(10,10))
plt.imshow(img)
plt.axis('off')
plt.show()
# 原图展示
test_img_path = "work/风格图像/虚空夜月.jpg"
img = mpimg.imread(test_img_path)
plt.figure(figsize=(10,10))
plt.imshow(img)
plt.axis('off')
plt.show()
# 预测结果展示
test_img_path = "transfer_result/ndarray_1620094320.1111157.jpg"
img = mpimg.imread(test_img_path)
# 展示预测结果图片
plt.figure(figsize=(10,10))
plt.imshow(img)
plt.show()```