001    package com.github.sarxos.webcam.ds.gstreamer;
002    
003    import java.awt.Dimension;
004    import java.awt.image.BufferedImage;
005    import java.awt.image.DataBufferInt;
006    import java.nio.IntBuffer;
007    import java.util.ArrayList;
008    import java.util.List;
009    import java.util.concurrent.TimeUnit;
010    import java.util.concurrent.atomic.AtomicBoolean;
011    
012    import org.gstreamer.Caps;
013    import org.gstreamer.Element;
014    import org.gstreamer.ElementFactory;
015    import org.gstreamer.Pad;
016    import org.gstreamer.Pipeline;
017    import org.gstreamer.State;
018    import org.gstreamer.Structure;
019    import org.gstreamer.elements.RGBDataSink;
020    import org.slf4j.Logger;
021    import org.slf4j.LoggerFactory;
022    
023    import com.github.sarxos.webcam.WebcamDevice;
024    import com.github.sarxos.webcam.WebcamResolution;
025    
026    
027    public class GStreamerDevice implements WebcamDevice, RGBDataSink.Listener, WebcamDevice.FPSSource {
028    
029            private static final Logger LOG = LoggerFactory.getLogger(GStreamerDevice.class);
030    
031            /**
032             * Limit the lateness of frames to no more than 20ms (half a frame at 25fps)
033             */
034            private static final long LATENESS = 20; // ms
035    
036            /**
037             * Video format to capture.
038             */
039            private static final String FORMAT = "video/x-raw-yuv";
040    
041            /**
042             * All possible resolutions - populated while initialization phase.
043             */
044            private Dimension[] resolutions = null;
045    
046            /**
047             * Device name, immutable.
048             */
049            private final String name;
050    
051            /* gstreamer stuff */
052    
053            private Pipeline pipe = null;
054            private Element source = null;
055            private Element filter = null;
056            private RGBDataSink sink = null;
057            private Caps caps = null;
058    
059            /* logic */
060    
061            private AtomicBoolean open = new AtomicBoolean(false);
062            private AtomicBoolean disposed = new AtomicBoolean(false);
063            private AtomicBoolean starting = new AtomicBoolean(false);
064            private AtomicBoolean initialized = new AtomicBoolean(false);
065            private Dimension resolution = WebcamResolution.VGA.getSize();
066            private BufferedImage image = null;
067    
068            /* used to calculate fps */
069    
070            private long t1 = -1;
071            private long t2 = -1;
072    
073            private volatile double fps = 0;
074    
075            /**
076             * Create GStreamer webcam device.
077             * 
078             * @param name the name of webcam device
079             */
080            protected GStreamerDevice(String name) {
081                    this.name = name;
082            }
083    
084            /**
085             * Initialize webcam device.
086             */
087            private synchronized void init() {
088    
089                    if (!initialized.compareAndSet(false, true)) {
090                            return;
091                    }
092    
093                    LOG.debug("GStreamer webcam device initialization");
094    
095                    pipe = new Pipeline(name);
096    
097                    source = ElementFactory.make("dshowvideosrc", "source");
098                    source.set("device-name", name);
099    
100                    sink = new RGBDataSink(name, this);
101                    sink.setPassDirectBuffer(true);
102                    sink.getSinkElement().setMaximumLateness(LATENESS, TimeUnit.MILLISECONDS);
103                    sink.getSinkElement().setQOSEnabled(true);
104    
105                    filter = ElementFactory.make("capsfilter", "filter");
106    
107                    resolutions = parseResolutions(source.getPads().get(0));
108            }
109    
110            /**
111             * Use GStreamer to get all possible resolutions.
112             * 
113             * @param pad the pad to get resolutions from
114             * @return Array of resolutions supported by device connected with pad
115             */
116            private static final Dimension[] parseResolutions(Pad pad) {
117    
118                    List<Dimension> dimensions = new ArrayList<Dimension>();
119    
120                    Caps caps = pad.getCaps();
121    
122                    Structure structure = null;
123                    String format = null;
124    
125                    int n = caps.size();
126                    int i = 0;
127    
128                    int w = -1;
129                    int h = -1;
130    
131                    do {
132    
133                            structure = caps.getStructure(i++);
134    
135                            LOG.debug("Found format structure {}", structure);
136    
137                            format = structure.getName();
138    
139                            if (format.equals(FORMAT)) {
140                                    w = structure.getRange("width").getMinInt();
141                                    h = structure.getRange("height").getMinInt();
142                                    dimensions.add(new Dimension(w, h));
143                            }
144    
145                    } while (i < n);
146    
147                    return dimensions.toArray(new Dimension[dimensions.size()]);
148            }
149    
150            @Override
151            public String getName() {
152                    return name;
153            }
154    
155            @Override
156            public Dimension[] getResolutions() {
157                    init();
158                    return resolutions;
159            }
160    
161            @Override
162            public Dimension getResolution() {
163                    return resolution;
164            }
165    
166            @Override
167            public void setResolution(Dimension size) {
168                    this.resolution = size;
169            }
170    
171            @Override
172            public BufferedImage getImage() {
173                    return image;
174            }
175    
176            @Override
177            public void open() {
178    
179                    if (!open.compareAndSet(false, true)) {
180                            return;
181                    }
182    
183                    LOG.debug("Opening GStreamer device");
184    
185                    init();
186    
187                    starting.set(true);
188    
189                    Dimension size = getResolution();
190    
191                    image = new BufferedImage(size.width, size.height, BufferedImage.TYPE_INT_RGB);
192                    image.setAccelerationPriority(0);
193                    image.flush();
194    
195                    if (caps != null) {
196                            caps.dispose();
197                    }
198    
199                    caps = Caps.fromString(String.format("%s,width=%d,height=%d", FORMAT, size.width, size.height));
200    
201                    filter.setCaps(caps);
202    
203                    LOG.debug("Link elements");
204    
205                    pipe.addMany(source, filter, sink);
206                    Element.linkMany(source, filter, sink);
207                    pipe.setState(State.PLAYING);
208    
209                    // wait max 20s for image to appear
210                    synchronized (this) {
211                            LOG.debug("Wait for device to be ready");
212                            try {
213                                    this.wait(20000);
214                            } catch (InterruptedException e) {
215                                    return;
216                            }
217                    }
218            }
219    
220            @Override
221            public void close() {
222    
223                    if (!open.compareAndSet(true, false)) {
224                            return;
225                    }
226    
227                    LOG.debug("Closing GStreamer device");
228    
229                    image = null;
230    
231                    LOG.debug("Unlink elements");
232    
233                    pipe.setState(State.NULL);
234                    Element.unlinkMany(source, filter, sink);
235                    pipe.removeMany(source, filter, sink);
236            }
237    
238            @Override
239            public void dispose() {
240    
241                    if (!disposed.compareAndSet(false, true)) {
242                            return;
243                    }
244    
245                    LOG.debug("Disposing GStreamer device");
246    
247                    close();
248    
249                    filter.dispose();
250                    source.dispose();
251                    sink.dispose();
252                    pipe.dispose();
253                    caps.dispose();
254            }
255    
256            @Override
257            public boolean isOpen() {
258                    return open.get();
259            }
260    
261            @Override
262            public void rgbFrame(boolean preroll, int width, int height, IntBuffer rgb) {
263    
264                    LOG.trace("New RGB frame");
265    
266                    if (t1 == -1 || t2 == -1) {
267                            t1 = System.currentTimeMillis();
268                            t2 = System.currentTimeMillis();
269                    }
270    
271                    BufferedImage tmp = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
272                    tmp.setAccelerationPriority(0);
273                    tmp.flush();
274    
275                    rgb.get(((DataBufferInt) tmp.getRaster().getDataBuffer()).getData(), 0, width * height);
276    
277                    image = tmp;
278    
279                    if (starting.compareAndSet(true, false)) {
280    
281                            synchronized (this) {
282                                    this.notifyAll();
283                            }
284    
285                            LOG.debug("GStreamer device ready");
286                    }
287    
288                    t1 = t2;
289                    t2 = System.currentTimeMillis();
290    
291                    fps = (4 * fps + 1000 / (t2 - t1 + 1)) / 5;
292            }
293    
294            @Override
295            public double getFPS() {
296                    return fps;
297            }
298    }