I’m trying to support a use case where a single func op contains a single region and single block with multiple ops that perform compute operations and pass data via tensors between them. So, the starting state is before any bufferization is done.
The compute ops will then be clustered into groups, and each group will be outlined to a different func op together with any required non compute ops.
After outlining, the module will be bufferized and the desired result should be such that data returned by one func and received by another should be placed on the same memory buffer and not on different memory buffers with copies between them.
A simple example of the initial state would be the following:
module {
func.func @main(%t: tensor<2xf32>) -> (tensor<4xf32>) {
%c = tensor.empty() : tensor<4xf32>
%a = tensor.empty() : tensor<2xf32>
%b = tensor.empty() : tensor<2xf32>
%1 = linalg.exp ins(%t : tensor<2xf32>) outs(%a: tensor<2xf32>) -> tensor<2xf32>
%2 = linalg.negf ins(%t : tensor<2xf32>) outs(%b: tensor<2xf32>) -> tensor<2xf32>
%inserted_slice = tensor.insert_slice %1 into %c[0] [2] [1] : tensor<2xf32> into tensor<4xf32>
%inserted_slice1 = tensor.insert_slice %2 into %inserted_slice[2] [2] [1] : tensor<2xf32> into tensor<4xf32>
%d = tensor.empty() : tensor<4xf32>
%3 = linalg.abs ins(%inserted_slice1 : tensor<4xf32>) outs(%d: tensor<4xf32>) -> tensor<4xf32>
return %3 : tensor<4xf32>
}
}
One possible desired output after bufferization would be the following:
module {
func.func @producer1(%arg0: memref<4xf32, 1>, %arg1: memref<2xf32, 1>) -> memref<4xf32, 1> {
%subview = memref.subview %arg0[0] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1]>, 1>
linalg.exp ins(%arg1 : memref<2xf32, 1>) outs(%subview : memref<2xf32, strided<[1]>, 1>)
return %arg0 : memref<4xf32, 1>
}
func.func @producer2(%arg0: memref<4xf32, 1>, %arg1: memref<2xf32, 1>) -> memref<4xf32, 1> {
%subview = memref.subview %arg0[2] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1], offset: 2>, 1>
linalg.negf ins(%arg1 : memref<2xf32, 1>) outs(%subview : memref<2xf32, strided<[1], offset: 2>, 1>)
return %arg0 : memref<4xf32, 1>
}
func.func @consumer(%arg0: memref<4xf32, 1>) -> memref<4xf32, 1> {
%alloc = memref.alloc() {alignment = 64 : i64} : memref<4xf32, 1>
linalg.abs ins(%arg0 : memref<4xf32, 1>) outs(%alloc : memref<4xf32, 1>)
return %alloc : memref<4xf32, 1>
}
func.func @main(%arg0: memref<2xf32, 1>) -> memref<4xf32, 1> {
%alloc = memref.alloc() {alignment = 64 : i64} : memref<4xf32, 1>
%0 = call @producer1(%alloc, %arg0) : (memref<4xf32, 1>, memref<2xf32, 1>) -> memref<4xf32, 1>
%1 = call @producer2(%0, %arg0) : (memref<4xf32, 1>, memref<2xf32, 1>) -> memref<4xf32, 1>
%2 = call @consumer(%1) : (memref<4xf32, 1>) -> memref<4xf32, 1>
return %2 : memref<4xf32, 1>
}
}
In this outcome the outputs of exp and negf and input to abs all use the same buffer.
On the contrary one possible non desired outcome would be:
module {
func.func private @producer1(%arg0: memref<2xf32, 1>) -> memref<2xf32, 1> {
%alloc = memref.alloc() {alignment = 64 : i64} : memref<2xf32, 1>
linalg.exp ins(%arg0 : memref<2xf32, 1>) outs(%alloc : memref<2xf32, 1>)
return %alloc : memref<2xf32, 1>
}
func.func private @producer2(%arg0: memref<2xf32, 1>) -> memref<2xf32, 1> {
%alloc = memref.alloc() {alignment = 64 : i64} : memref<2xf32, 1>
linalg.negf ins(%arg0 : memref<2xf32, 1>) outs(%alloc : memref<2xf32, 1>)
return %alloc : memref<2xf32, 1>
}
func.func private @consumer(%arg0: memref<2xf32, 1>, %arg1: memref<2xf32, 1>) -> memref<4xf32, 1> {
%alloc = memref.alloc() {alignment = 64 : i64} : memref<4xf32, 1>
%alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<4xf32, 1>
%subview = memref.subview %alloc[0] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1]>, 1>
memref.copy %arg0, %subview : memref<2xf32, 1> to memref<2xf32, strided<[1]>, 1>
%subview_1 = memref.subview %alloc[2] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1], offset: 2>, 1>
memref.copy %arg1, %subview_1 : memref<2xf32, 1> to memref<2xf32, strided<[1], offset: 2>, 1>
linalg.abs ins(%alloc : memref<4xf32, 1>) outs(%alloc_0 : memref<4xf32, 1>)
return %alloc_0 : memref<4xf32, 1>
}
func.func @main(%arg0: memref<2xf32, 1>) -> memref<4xf32, 1> {
%0 = call @producer1(%arg0) : (memref<2xf32, 1>) -> memref<2xf32, 1>
%1 = call @producer2(%arg0) : (memref<2xf32, 1>) -> memref<2xf32, 1>
%2 = call @consumer(%0, %1) : (memref<2xf32, 1>, memref<2xf32, 1>) -> memref<4xf32, 1>
return %2 : memref<4xf32, 1>
}
}
In this case they do not use the same buffer and copies are done.
I’ve been trying to understand if it is possible to use the one shot bufferize pass to achieve the desired result, and if so how the outline step should be done. Another goal is to try to have the outline step as simple as possible.
To this end I made several experiments with the one shot bufferize pass via different outlining options.
The simplest outlining option I tried was to outline each group with all other non compute ops it depends on:
module {
func.func @producer1(%arg0: tensor<2xf32>) -> (tensor<2xf32> ) {
%a = tensor.empty() : tensor<2xf32>
%1 = linalg.exp ins(%arg0 : tensor<2xf32>) outs(%a: tensor<2xf32>) -> tensor<2xf32>
return %1 : tensor<2xf32>
}
func.func @producer2(%arg0: tensor<2xf32>) -> (tensor<2xf32> ) {
%b = tensor.empty() : tensor<2xf32>
%1 = linalg.negf ins(%arg0 : tensor<2xf32>) outs(%b: tensor<2xf32>) -> tensor<2xf32>
return %1 : tensor<2xf32>
}
func.func @consumer(%arg0: tensor<2xf32>, %arg1: tensor<2xf32>) -> (tensor<4xf32> ) {
%0 = tensor.empty() : tensor<4xf32>
%1 = tensor.empty() : tensor<4xf32>
%inserted_slice = tensor.insert_slice %arg0 into %0[0] [2] [1] : tensor<2xf32> into tensor<4xf32>
%inserted_slice1 = tensor.insert_slice %arg1 into %inserted_slice[2] [2] [1] : tensor<2xf32> into tensor<4xf32>
%2 = linalg.abs ins(%inserted_slice1 : tensor<4xf32>) outs(%1: tensor<4xf32>) -> tensor<4xf32>
return %2 : tensor<4xf32>
}
func.func @main(%t: tensor<2xf32>) -> (tensor<4xf32>) {
%0 = call @producer1(%t) : (tensor<2xf32>) -> (tensor<2xf32>)
%1 = call @producer2(%t) : (tensor<2xf32>) -> (tensor<2xf32>)
%2 = call @consumer(%0, %1) : (tensor<2xf32>, tensor<2xf32>) -> (tensor<4xf32>)
return %2 : tensor<4xf32>
}
}
The outcome of bufferization is not desirable since exp negf and abs do not share the same buffer:
module {
func.func @producer1(%arg0: memref<2xf32, 1>) -> memref<2xf32, 1> {
%alloc = memref.alloc() {alignment = 64 : i64} : memref<2xf32, 1>
linalg.exp ins(%arg0 : memref<2xf32, 1>) outs(%alloc : memref<2xf32, 1>)
return %alloc : memref<2xf32, 1>
}
func.func @producer2(%arg0: memref<2xf32, 1>) -> memref<2xf32, 1> {
%alloc = memref.alloc() {alignment = 64 : i64} : memref<2xf32, 1>
linalg.negf ins(%arg0 : memref<2xf32, 1>) outs(%alloc : memref<2xf32, 1>)
return %alloc : memref<2xf32, 1>
}
func.func @consumer(%arg0: memref<2xf32, 1>, %arg1: memref<2xf32, 1>) -> memref<4xf32, 1> {
%alloc = memref.alloc() {alignment = 64 : i64} : memref<4xf32, 1>
%alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<4xf32, 1>
%subview = memref.subview %alloc[0] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1]>, 1>
memref.copy %arg0, %subview : memref<2xf32, 1> to memref<2xf32, strided<[1]>, 1>
%subview_1 = memref.subview %alloc[2] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1], offset: 2>, 1>
memref.copy %arg1, %subview_1 : memref<2xf32, 1> to memref<2xf32, strided<[1], offset: 2>, 1>
linalg.abs ins(%alloc : memref<4xf32, 1>) outs(%alloc_0 : memref<4xf32, 1>)
return %alloc_0 : memref<4xf32, 1>
}
func.func @main(%arg0: memref<2xf32, 1>) -> memref<4xf32, 1> {
%0 = call @producer1(%arg0) : (memref<2xf32, 1>) -> memref<2xf32, 1>
%1 = call @producer2(%arg0) : (memref<2xf32, 1>) -> memref<2xf32, 1>
%2 = call @consumer(%0, %1) : (memref<2xf32, 1>, memref<2xf32, 1>) -> memref<4xf32, 1>
return %2 : memref<4xf32, 1>
}
}
Another option I tried was to replace the small tensor.empty ops with an extract from the large one similar to what is done in eliminate empty tensors pass, and outline in a less trivial way before bufferization:
module {
func.func @producer1(%arg0: tensor<2xf32>, %buffer: tensor<4xf32>) -> (tensor<4xf32> ) {
%extracted_slice = tensor.extract_slice %buffer[0] [2] [1] : tensor<4xf32> to tensor<2xf32>
%1 = linalg.exp ins(%arg0 : tensor<2xf32>) outs(%extracted_slice: tensor<2xf32>) -> tensor<2xf32>
%inserted_slice = tensor.insert_slice %1 into %buffer[0] [2] [1] : tensor<2xf32> into tensor<4xf32>
return %inserted_slice : tensor<4xf32>
}
func.func @producer2(%arg0: tensor<2xf32>, %buffer: tensor<4xf32>) -> (tensor<4xf32> ) {
%extracted_slice = tensor.extract_slice %buffer[2] [2] [1] : tensor<4xf32> to tensor<2xf32>
%1 = linalg.exp ins(%arg0 : tensor<2xf32>) outs(%extracted_slice: tensor<2xf32>) -> tensor<2xf32>
%inserted_slice = tensor.insert_slice %1 into %buffer[2] [2] [1] : tensor<2xf32> into tensor<4xf32>
return %inserted_slice : tensor<4xf32>
}
func.func @consumer(%buffer: tensor<4xf32>) -> (tensor<4xf32> ) {
%0 = tensor.empty() : tensor<4xf32>
%1 = linalg.abs ins(%buffer : tensor<4xf32>) outs(%0: tensor<4xf32>) -> tensor<4xf32>
return %1 : tensor<4xf32>
}
func.func @main(%t: tensor<2xf32>) -> (tensor<4xf32>) {
%buffer = tensor.empty() : tensor<4xf32>
%0 = call @producer1(%t, %buffer) : (tensor<2xf32>, tensor<4xf32>) -> (tensor<4xf32>)
%1 = call @producer2(%t, %0) : (tensor<2xf32>, tensor<4xf32>) -> (tensor<4xf32>)
%2 = call @consumer(%1) : (tensor<4xf32>) -> (tensor<4xf32>)
return %2 : tensor<4xf32>
}
}
In this case the output of bufferization did lead to a single buffer, but there are additional copies that were generated:
module {
func.func @producer1(%arg0: memref<2xf32, 1>, %arg1: memref<4xf32, 1>) -> memref<4xf32, 1> {
%subview = memref.subview %arg1[0] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1]>, 1>
linalg.exp ins(%arg0 : memref<2xf32, 1>) outs(%subview : memref<2xf32, strided<[1]>, 1>)
%subview_0 = memref.subview %arg1[0] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1]>, 1>
memref.copy %subview, %subview_0 : memref<2xf32, strided<[1]>, 1> to memref<2xf32, strided<[1]>, 1>
return %arg1 : memref<4xf32, 1>
}
func.func @producer2(%arg0: memref<2xf32, 1>, %arg1: memref<4xf32, 1>) -> memref<4xf32, 1> {
%subview = memref.subview %arg1[2] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1], offset: 2>, 1>
linalg.exp ins(%arg0 : memref<2xf32, 1>) outs(%subview : memref<2xf32, strided<[1], offset: 2>, 1>)
%subview_0 = memref.subview %arg1[2] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1], offset: 2>, 1>
memref.copy %subview, %subview_0 : memref<2xf32, strided<[1], offset: 2>, 1> to memref<2xf32, strided<[1], offset: 2>, 1>
return %arg1 : memref<4xf32, 1>
}
func.func @consumer(%arg0: memref<4xf32, 1>) -> memref<4xf32, 1> {
%alloc = memref.alloc() {alignment = 64 : i64} : memref<4xf32, 1>
linalg.abs ins(%arg0 : memref<4xf32, 1>) outs(%alloc : memref<4xf32, 1>)
return %alloc : memref<4xf32, 1>
}
func.func @main(%arg0: memref<2xf32, 1>) -> memref<4xf32, 1> {
%alloc = memref.alloc() {alignment = 64 : i64} : memref<4xf32, 1>
%0 = call @producer1(%arg0, %alloc) : (memref<2xf32, 1>, memref<4xf32, 1>) -> memref<4xf32, 1>
%1 = call @producer2(%arg0, %0) : (memref<2xf32, 1>, memref<4xf32, 1>) -> memref<4xf32, 1>
%2 = call @consumer(%1) : (memref<4xf32, 1>) -> memref<4xf32, 1>
return %2 : memref<4xf32, 1>
}
}
An additional experiment I tried was to just bufferize without outlining, and the outcome does include the undesired extra buffers and copies:
module {
func.func @main(%arg0: memref<2xf32, 1>) -> memref<4xf32, 1> {
%alloc = memref.alloc() {alignment = 64 : i64} : memref<4xf32, 1>
%alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<2xf32, 1>
%alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<2xf32, 1>
linalg.exp ins(%arg0 : memref<2xf32, 1>) outs(%alloc_0 : memref<2xf32, 1>)
linalg.negf ins(%arg0 : memref<2xf32, 1>) outs(%alloc_1 : memref<2xf32, 1>)
%subview = memref.subview %alloc[0] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1]>, 1>
memref.copy %alloc_0, %subview : memref<2xf32, 1> to memref<2xf32, strided<[1]>, 1>
%subview_2 = memref.subview %alloc[2] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1], offset: 2>, 1>
memref.copy %alloc_1, %subview_2 : memref<2xf32, 1> to memref<2xf32, strided<[1], offset: 2>, 1>
%alloc_3 = memref.alloc() {alignment = 64 : i64} : memref<4xf32, 1>
linalg.abs ins(%alloc : memref<4xf32, 1>) outs(%alloc_3 : memref<4xf32, 1>)
return %alloc_3 : memref<4xf32, 1>
}
}
I also tried to first eliminate the small tensor.empty ops before bufferizing:
module {
func.func @main(%arg0: tensor<2xf32>) -> tensor<4xf32> {
%0 = tensor.empty() : tensor<4xf32>
%extracted_slice = tensor.extract_slice %0[0] [2] [1] : tensor<4xf32> to tensor<2xf32>
%extracted_slice1 = tensor.extract_slice %0[2] [2] [1] : tensor<4xf32> to tensor<2xf32>
%2 = linalg.exp ins(%arg0 : tensor<2xf32>) outs(%extracted_slice : tensor<2xf32>) -> tensor<2xf32>
%3 = linalg.negf ins(%arg0 : tensor<2xf32>) outs(%extracted_slice1 : tensor<2xf32>) -> tensor<2xf32>
%inserted_slice = tensor.insert_slice %2 into %0[0] [2] [1] : tensor<2xf32> into tensor<4xf32>
%inserted_slice_0 = tensor.insert_slice %3 into %inserted_slice[2] [2] [1] : tensor<2xf32> into tensor<4xf32>
%4 = tensor.empty() : tensor<4xf32>
%5 = linalg.abs ins(%inserted_slice_0 : tensor<4xf32>) outs(%4 : tensor<4xf32>) -> tensor<4xf32>
return %5 : tensor<4xf32>
}
}
In this case the outcome included one additional small buffer and 2 copies:
module {
func.func @main(%arg0: memref<2xf32, 1>) -> memref<4xf32, 1> {
%alloc = memref.alloc() {alignment = 64 : i64} : memref<4xf32, 1>
%subview = memref.subview %alloc[0] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1]>, 1>
%subview_0 = memref.subview %alloc[2] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1], offset: 2>, 1>
%alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<2xf32, 1>
linalg.exp ins(%arg0 : memref<2xf32, 1>) outs(%subview : memref<2xf32, strided<[1]>, 1>)
linalg.negf ins(%arg0 : memref<2xf32, 1>) outs(%alloc_1 : memref<2xf32, 1>)
%subview_2 = memref.subview %alloc[0] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1]>, 1>
memref.copy %subview, %subview_2 : memref<2xf32, strided<[1]>, 1> to memref<2xf32, strided<[1]>, 1>
%subview_3 = memref.subview %alloc[2] [2] [1] : memref<4xf32, 1> to memref<2xf32, strided<[1], offset: 2>, 1>
memref.copy %alloc_1, %subview_3 : memref<2xf32, 1> to memref<2xf32, strided<[1], offset: 2>, 1>
%alloc_4 = memref.alloc() {alignment = 64 : i64} : memref<4xf32, 1>
linalg.abs ins(%alloc : memref<4xf32, 1>) outs(%alloc_4 : memref<4xf32, 1>)
return %alloc_4 : memref<4xf32, 1>
}
}
Any assistance in understanding what analysis the one shot bufferize pass can do to help with sharing buffers between func ops and reducing copies would be greatly appreciated. @matthias-springer