Netty: TCP file transfer doesn’t work correctly

I am working on my online file storage and today I have encountered some issues with my Netty TCP file tranfer. So the problem is that only 8192 bytes of data is actually written in the file on the client-side. I want to know what the problem is, and how I can fix it.

I have seen ALL of the other (5) stackoverflow questions.

Here is my server bootstrap:

package com.martin.main;

import com.martin.file.*;
import com.martin.handler.*;
import com.martin.utils.*;
import io.netty.bootstrap.*;
import io.netty.channel.*;
import io.netty.channel.nio.*;
import io.netty.channel.socket.*;
import io.netty.channel.socket.nio.*;
import io.netty.handler.codec.*;

import java.io.*;
import java.net.*;
import java.util.*;
import java.util.concurrent.*;

public class Server {

    public static void main(String[] args) {
        new Server().setup(4783);
    }

    public ChannelFuture setup(int port) {
        ChannelFuture future = null;
        EventLoopGroup bossGroup = new NioEventLoopGroup();
        EventLoopGroup workerGroup = new NioEventLoopGroup();
        final ServerBootstrap bootstrap;

     
        try {
            bootstrap = new ServerBootstrap();
            bootstrap.group(bossGroup, workerGroup);
            bootstrap.channel(NioServerSocketChannel.class);
            bootstrap.childHandler(new ChannelInitializer<SocketChannel>() {

                @Override
                protected void initChannel(SocketChannel socketChannel) throws Exception {
                    ChannelPipeline pipeline = socketChannel.pipeline();
                    //pipeline.addLast("framer", new LengthFieldBasedFrameDecoder());
                    pipeline.addLast("framer", new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4));
                    pipeline.addLast(new LengthFieldPrepender(4));
                    pipeline.addLast("login", new LoginServerHandler()); //the problem is not in that handler
                }
            });
            future = bootstrap.bind(new InetSocketAddress(port)).sync().channel().closeFuture().sync();
        }catch(InterruptedException e) {
            e.printStackTrace();
        } finally {
            bossGroup.shutdownGracefully();
            workerGroup.shutdownGracefully();
        }
        return future;
    }

}

My server-handler:

public class ServerHandler extends ByteToMessageDecoder {

    @Override
    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
        cause.printStackTrace();
    }

    @Override
    protected void decode(ChannelHandlerContext ctx, ByteBuf byteBuf, List<Object> list) throws Exception {

        byte msgType = byteBuf.readByte();

        if(FILEPACKETINFO == msgType) {
            //The problem is probaly here
            System.out.println("inide file req.");
            int id = byteBuf.readInt();

            for(OnlineFile onlineFile : OnlineFile.getOnlineFiles()) {
                if(onlineFile.getId() == id) {
                    System.out.println("file request id: " + id + " actual id: " + onlineFile.getId());
                    ByteBuf buf = Unpooled.buffer();
                    buf.writeByte(FILEPACKETINFO);
                    String fileName = onlineFile.getName();
                    File file = onlineFile.getActualFile();
                    buf.writeLong(file.length());
                    buf.writeInt(fileName.length());
                    buf.writeCharSequence(fileName, CharsetUtil.US_ASCII);
                    ctx.writeAndFlush(buf);

                    if(!(file.length() <= 0)) {

                        ctx.writeAndFlush(new ChunkedFile(file)).addListener(new ChannelFutureListener() {
                            @Override
                            public void operationComplete(ChannelFuture channelFuture) throws Exception {
                                System.out.println("written successfully!");
                                if (channelFuture.cause() != null) channelFuture.cause().printStackTrace();
                            }
                        });
                    } else {
                        System.out.println("length 0: " + " (" + file.length() +")");
                    }
                }
            }
        } else if(REQUESTVISUALFILES == msgType) {
            createAndSend(ctx); //send VISUAL files, problem not there
        } else if(REQUESTFOLDERFILESBYID == msgType) {
            int id = byteBuf.readInt();
            createAndSend(ctx, id); //send VISUAL files by id, problem not there
        }
    }

My client bootstrap:

    public static ChannelFuture setup(String host, int port) {
        ChannelFuture channelFuture = null;

        EventLoopGroup group = new NioEventLoopGroup();

        try {
            Bootstrap bootstrap = new Bootstrap();
            bootstrap.group(group);
            bootstrap.channel(NioSocketChannel.class);
            bootstrap.handler(new ChannelInitializer<SocketChannel>() {
                @Override
                protected void initChannel(SocketChannel socketChannel) throws Exception {
                    ChannelPipeline pipeline = socketChannel.pipeline();
                    pipeline.addLast("framer", new LengthFieldBasedFrameDecoder(64*1024, 0, 4, 0, 4));
                    pipeline.addLast(lfp);
                    pipeline.addLast("login", new LoginHandler()); //the problem is not in that handler either
                }
            });
            channelFuture = bootstrap.connect(new InetSocketAddress(host, port)).sync();
            System.out.println("the setup came to an end!");
            LoginForm.createAndShowGUI("Login");
            mainChannel = channelFuture.channel();
            channelFuture.channel().closeFuture().sync();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } finally {
            group.shutdownGracefully();
        }
        return channelFuture;
    }

}

My client handler:

package com.martin.handler;

import io.netty.buffer.*;
import io.netty.channel.*;
import io.netty.util.*;

import java.io.*;
import java.nio.*;
import java.nio.channels.*;

public class FileChunkHandler extends SimpleChannelInboundHandler<ByteBuf> {

    public static String currentFileName = "Test.txt";

    @Override
    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
        cause.printStackTrace();
        ctx.pipeline().remove(this);
    }

    @Override
    protected void channelRead0(ChannelHandlerContext ctx, ByteBuf byteBuf) throws Exception {
        //the problem is probably here
        System.out.println("inside FileChunkHandler current file: " + currentFileName);
        ByteBuffer buffer = byteBuf.nioBuffer();
        System.out.println(buffer.capacity() + " buffer" + " bytebuf: " + byteBuf.readableBytes());

        File file = triggerFileCreation();

       // FileOutputStream fos = new FileOutputStream("C:\Users\marti\storage\" + currentFileName);
        RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw");
        FileChannel channel = randomAccessFile.getChannel();

        while(buffer.hasRemaining()) {
            channel.position(file.length());
            channel.write(buffer);
            System.out.println("chunk has just been written");
        }
        channel.close();
        randomAccessFile.close();

        //!!!--->IMPORTANT<-----!!!
        ctx.pipeline().remove(this);
    }
    public static File triggerFileCreation() {
        File file = new File(System.getProperty("user.home") + "/storage/" + currentFileName);
        if(!file.exists()) {
            try {
                file.createNewFile();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return file;
    }
}

EDIT

The problem was that I removed the handler too quickly, then other packets got handled in other handlers/were dropped from the pipeline.

Answer

The problem is indeed most likely to be in your FileChunkHandler.

You only read a single buffer (likely containing 8192 bytes – 8kB), and then remove the handler. The remaining chunks either get “handled” by some other handler in the pipeline, or reach the end of the pipeline and get dropped. As mentioned in Discord, you need to keep track of how many bytes you are expecting, subtract the number received, and only when that number reaches 0, should you remove the handler.